sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 import arvados_cwl.util
   8 #from arvados_cwl.arvdocker import arv_docker_clear_cache
   9 import copy
  10 import arvados.config
  11 import logging
  12 import unittest
  13 import os
  14 import functools
  15 import threading
  16 import cwltool.process
  17 import cwltool.secrets
  18 import cwltool.load_tool
  19 from cwltool.update import INTERNAL_VERSION
  20 from schema_salad.ref_resolver import Loader
  21 from schema_salad.sourceline import cmap
  22 import io
  23
  24 from unittest import mock
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         #arv_docker_clear_cache()
  64
  65     def tearDown(self):
  66         root_logger = logging.getLogger('')
  67
  68         # Remove existing RuntimeStatusLoggingHandlers if they exist
  69         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  70         root_logger.handlers = handlers
  71
  72     def helper(self, runner, enable_reuse=True):
  73         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  74
  75         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  76                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  77         fs_access = mock.MagicMock()
  78         fs_access.exists.return_value = True
  79
  80         loadingContext = arvados_cwl.context.ArvLoadingContext(
  81             {"avsc_names": avsc_names,
  82              "basedir": "",
  83              "make_fs_access": make_fs_access,
  84              "construct_tool_object": runner.arv_make_tool,
  85              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  86              "loader": Loader({}),
  87              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  88              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             #arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 139
 140             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 141             runner.api.collections().get().execute.return_value = {
 142                 "portable_data_hash": "99999999999999999999999999999993+99"}
 143
 144             tool = cmap({
 145                 "inputs": [],
 146                 "outputs": [],
 147                 "baseCommand": "ls",
 148                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 149                 "id": "",
 150                 "class": "CommandLineTool",
 151                 "cwlVersion": "v1.2"
 152             })
 153
 154             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 155
 156             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 157             arvtool.formatgraph = None
 158
 159             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 160                 j.run(runtimeContext)
 161                 runner.api.container_requests().create.assert_called_with(
 162                     body=JsonDiffMatcher({
 163                         'environment': {
 164                             'HOME': '/var/spool/cwl',
 165                             'TMPDIR': '/tmp'
 166                         },
 167                         'name': 'test_run_'+str(enable_reuse),
 168                         'runtime_constraints': {
 169                             'vcpus': 1,
 170                             'ram': 268435456
 171                         },
 172                         'use_existing': enable_reuse,
 173                         'priority': 500,
 174                         'mounts': {
 175                             '/tmp': {'kind': 'tmp',
 176                                      "capacity": 1073741824
 177                                  },
 178                             '/var/spool/cwl': {'kind': 'tmp',
 179                                                "capacity": 1073741824 }
 180                         },
 181                         'state': 'Committed',
 182                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 183                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                         'output_path': '/var/spool/cwl',
 185                         'output_ttl': 0,
 186                         'container_image': '99999999999999999999999999999993+99',
 187                         'command': ['ls', '/var/spool/cwl'],
 188                         'cwd': '/var/spool/cwl',
 189                         'scheduling_parameters': {},
 190                         'properties': {'cwl_input': {}},
 191                         'secret_mounts': {},
 192                         'output_storage_classes': ["default"]
 193                     }))
 194
 195     # The test passes some fields in builder.resources
 196     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 198     def test_resource_requirements(self, keepdocker):
 199         arvados_cwl.add_arv_hints()
 200         runner = mock.MagicMock()
 201         runner.ignore_docker_for_reuse = False
 202         runner.intermediate_output_ttl = 3600
 203         runner.secret_store = cwltool.secrets.SecretStore()
 204         runner.api._rootDesc = {"revision": "20210628"}
 205         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 206
 207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 208         runner.api.collections().get().execute.return_value = {
 209             "portable_data_hash": "99999999999999999999999999999993+99"}
 210
 211         tool = cmap({
 212             "inputs": [],
 213             "outputs": [],
 214             "hints": [{
 215                 "class": "ResourceRequirement",
 216                 "coresMin": 3,
 217                 "ramMin": 3000,
 218                 "tmpdirMin": 4000,
 219                 "outdirMin": 5000
 220             }, {
 221                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 222                 "keep_cache": 512
 223             }, {
 224                 "class": "http://arvados.org/cwl#APIRequirement",
 225             }, {
 226                 "class": "http://arvados.org/cwl#PartitionRequirement",
 227                 "partition": "blurb"
 228             }, {
 229                 "class": "http://arvados.org/cwl#IntermediateOutput",
 230                 "outputTTL": 7200
 231             }, {
 232                 "class": "WorkReuse",
 233                 "enableReuse": False
 234             }],
 235             "baseCommand": "ls",
 236             "id": "",
 237             "class": "CommandLineTool",
 238             "cwlVersion": "v1.2"
 239         })
 240
 241         loadingContext, runtimeContext = self.helper(runner)
 242         runtimeContext.name = "test_resource_requirements"
 243
 244         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 245         arvtool.formatgraph = None
 246         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 247             j.run(runtimeContext)
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         call_body_expected = {
 252             'environment': {
 253                 'HOME': '/var/spool/cwl',
 254                 'TMPDIR': '/tmp'
 255             },
 256             'name': 'test_resource_requirements',
 257             'runtime_constraints': {
 258                 'vcpus': 3,
 259                 'ram': 3145728000,
 260                 'keep_cache_ram': 536870912,
 261                 'API': True
 262             },
 263             'use_existing': False,
 264             'priority': 500,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 4194304000 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 5242880000 }
 270             },
 271             'state': 'Committed',
 272             'output_name': 'Output from step test_resource_requirements',
 273             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 274             'output_path': '/var/spool/cwl',
 275             'output_ttl': 7200,
 276             'container_image': '99999999999999999999999999999993+99',
 277             'command': ['ls'],
 278             'cwd': '/var/spool/cwl',
 279             'scheduling_parameters': {
 280                 'partitions': ['blurb']
 281             },
 282             'properties': {'cwl_input': {}},
 283             'secret_mounts': {},
 284             'output_storage_classes': ["default"]
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # The test passes some fields in builder.resources
 294     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 296     @mock.patch("arvados.collection.Collection")
 297     def test_initial_work_dir(self, collection_mock, keepdocker):
 298         runner = mock.MagicMock()
 299         runner.ignore_docker_for_reuse = False
 300         runner.intermediate_output_ttl = 0
 301         runner.secret_store = cwltool.secrets.SecretStore()
 302         runner.api._rootDesc = {"revision": "20210628"}
 303         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 304
 305         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 306         runner.api.collections().get().execute.return_value = {
 307             "portable_data_hash": "99999999999999999999999999999993+99"}
 308
 309         sourcemock = mock.MagicMock()
 310         def get_collection_mock(p):
 311             if "/" in p:
 312                 return (sourcemock, p.split("/", 1)[1])
 313             else:
 314                 return (sourcemock, "")
 315         runner.fs_access.get_collection.side_effect = get_collection_mock
 316
 317         vwdmock = mock.MagicMock()
 318         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 319
 320         tool = cmap({
 321             "inputs": [],
 322             "outputs": [],
 323             "hints": [{
 324                 "class": "InitialWorkDirRequirement",
 325                 "listing": [{
 326                     "class": "File",
 327                     "basename": "foo",
 328                     "location": "keep:99999999999999999999999999999995+99/bar"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "foo2",
 333                     "location": "keep:99999999999999999999999999999995+99"
 334                 },
 335                 {
 336                     "class": "File",
 337                     "basename": "filename",
 338                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 339                 },
 340                 {
 341                     "class": "Directory",
 342                     "basename": "subdir",
 343                     "location": "keep:99999999999999999999999999999995+99/subdir"
 344                 }                        ]
 345             }],
 346             "baseCommand": "ls",
 347             "class": "CommandLineTool",
 348             "cwlVersion": "v1.2",
 349             "id": ""
 350         })
 351
 352         loadingContext, runtimeContext = self.helper(runner)
 353         runtimeContext.name = "test_initial_work_dir"
 354
 355         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 356
 357         arvtool.formatgraph = None
 358         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 359             j.run(runtimeContext)
 360
 361         call_args, call_kwargs = runner.api.container_requests().create.call_args
 362
 363         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 367
 368         call_body_expected = {
 369             'environment': {
 370                 'HOME': '/var/spool/cwl',
 371                 'TMPDIR': '/tmp'
 372             },
 373             'name': 'test_initial_work_dir',
 374             'runtime_constraints': {
 375                 'vcpus': 1,
 376                 'ram': 268435456
 377             },
 378             'use_existing': True,
 379             'priority': 500,
 380             'mounts': {
 381                 '/tmp': {'kind': 'tmp',
 382                          "capacity": 1073741824 },
 383                 '/var/spool/cwl': {'kind': 'tmp',
 384                                    "capacity": 1073741824 },
 385                 '/var/spool/cwl/foo': {
 386                     'kind': 'collection',
 387                     'path': 'foo',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/foo2': {
 391                     'kind': 'collection',
 392                     'path': 'foo2',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 },
 395                 '/var/spool/cwl/filename': {
 396                     'kind': 'collection',
 397                     'path': 'filename',
 398                     'portable_data_hash': '99999999999999999999999999999996+99'
 399                 },
 400                 '/var/spool/cwl/subdir': {
 401                     'kind': 'collection',
 402                     'path': 'subdir',
 403                     'portable_data_hash': '99999999999999999999999999999996+99'
 404                 }
 405             },
 406             'state': 'Committed',
 407             'output_name': 'Output from step test_initial_work_dir',
 408             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409             'output_path': '/var/spool/cwl',
 410             'output_ttl': 0,
 411             'container_image': '99999999999999999999999999999993+99',
 412             'command': ['ls'],
 413             'cwd': '/var/spool/cwl',
 414             'scheduling_parameters': {
 415             },
 416             'properties': {'cwl_input': {}},
 417             'secret_mounts': {},
 418             'output_storage_classes': ["default"]
 419         }
 420
 421         call_body = call_kwargs.get('body', None)
 422         self.assertNotEqual(None, call_body)
 423         for key in call_body:
 424             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 425
 426
 427     # Test redirecting stdin/stdout/stderr
 428     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 429     def test_redirects(self, keepdocker):
 430         runner = mock.MagicMock()
 431         runner.ignore_docker_for_reuse = False
 432         runner.intermediate_output_ttl = 0
 433         runner.secret_store = cwltool.secrets.SecretStore()
 434         runner.api._rootDesc = {"revision": "20210628"}
 435         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 436
 437         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 438         runner.api.collections().get().execute.return_value = {
 439             "portable_data_hash": "99999999999999999999999999999993+99"}
 440
 441         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 442
 443         tool = cmap({
 444             "inputs": [],
 445             "outputs": [],
 446             "baseCommand": "ls",
 447             "stdout": "stdout.txt",
 448             "stderr": "stderr.txt",
 449             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 450             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 451             "id": "",
 452             "class": "CommandLineTool",
 453             "cwlVersion": "v1.2"
 454         })
 455
 456         loadingContext, runtimeContext = self.helper(runner)
 457         runtimeContext.name = "test_run_redirect"
 458
 459         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 460         arvtool.formatgraph = None
 461         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 462             j.run(runtimeContext)
 463             runner.api.container_requests().create.assert_called_with(
 464                 body=JsonDiffMatcher({
 465                     'environment': {
 466                         'HOME': '/var/spool/cwl',
 467                         'TMPDIR': '/tmp'
 468                     },
 469                     'name': 'test_run_redirect',
 470                     'runtime_constraints': {
 471                         'vcpus': 1,
 472                         'ram': 268435456
 473                     },
 474                     'use_existing': True,
 475                     'priority': 500,
 476                     'mounts': {
 477                         '/tmp': {'kind': 'tmp',
 478                                  "capacity": 1073741824 },
 479                         '/var/spool/cwl': {'kind': 'tmp',
 480                                            "capacity": 1073741824 },
 481                         "stderr": {
 482                             "kind": "file",
 483                             "path": "/var/spool/cwl/stderr.txt"
 484                         },
 485                         "stdin": {
 486                             "kind": "collection",
 487                             "path": "file.txt",
 488                             "portable_data_hash": "99999999999999999999999999999996+99"
 489                         },
 490                         "stdout": {
 491                             "kind": "file",
 492                             "path": "/var/spool/cwl/stdout.txt"
 493                         },
 494                     },
 495                     'state': 'Committed',
 496                     "output_name": "Output from step test_run_redirect",
 497                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 498                     'output_path': '/var/spool/cwl',
 499                     'output_ttl': 0,
 500                     'container_image': '99999999999999999999999999999993+99',
 501                     'command': ['ls', '/var/spool/cwl'],
 502                     'cwd': '/var/spool/cwl',
 503                     'scheduling_parameters': {},
 504                     'properties': {'cwl_input': {}},
 505                     'secret_mounts': {},
 506                     'output_storage_classes': ["default"]
 507                 }))
 508
 509     @mock.patch("arvados.collection.Collection")
 510     def test_done(self, col):
 511         api = mock.MagicMock()
 512
 513         runner = mock.MagicMock()
 514         runner.api = api
 515         runner.num_retries = 0
 516         runner.ignore_docker_for_reuse = False
 517         runner.intermediate_output_ttl = 0
 518         runner.secret_store = cwltool.secrets.SecretStore()
 519
 520         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 521
 522         runner.api.containers().get().execute.return_value = {"state":"Complete",
 523                                                               "output": "abc+123",
 524                                                               "exit_code": 0}
 525
 526         # Need to noop-out the close method otherwise it gets
 527         # discarded when closed and we can't call getvalue() to check
 528         # it.
 529         class NoopCloseStringIO(io.StringIO):
 530             def close(self):
 531                 pass
 532
 533         usage_report = NoopCloseStringIO()
 534         def colreader_action(name, mode):
 535             nonlocal usage_report
 536             if name == "node.json":
 537                 return io.StringIO("""{
 538     "ProviderType": "c5.large",
 539     "VCPUs": 2,
 540     "RAM": 4294967296,
 541     "IncludedScratch": 8000000000000,
 542     "AddedScratch": 0,
 543     "Price": 0.085,
 544     "Preemptible": false,
 545     "CUDA": {
 546         "DriverVersion": "",
 547         "HardwareCapability": "",
 548         "DeviceCount": 0
 549     }
 550 }""")
 551             if name == 'crunchstat.txt':
 552                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 553             if name == 'arv-mount.txt':
 554                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 555             if name == 'usage_report.html':
 556                 return usage_report
 557             return None
 558
 559         col().open.side_effect = colreader_action
 560         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 561
 562         loadingContext, runtimeContext = self.helper(runner)
 563
 564         arvjob = arvados_cwl.ArvadosContainer(runner,
 565                                               runtimeContext,
 566                                               mock.MagicMock(),
 567                                               {},
 568                                               None,
 569                                               [],
 570                                               [],
 571                                               "testjob")
 572         arvjob.output_callback = mock.MagicMock()
 573         arvjob.collect_outputs = mock.MagicMock()
 574         arvjob.successCodes = [0]
 575         arvjob.outdir = "/var/spool/cwl"
 576         arvjob.output_ttl = 3600
 577         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 578
 579         arvjob.collect_outputs.return_value = {"out": "stuff"}
 580
 581         arvjob.done({
 582             "state": "Final",
 583             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 584             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 585             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 586             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 587             "modified_at": "2017-05-26T12:01:22Z",
 588             "properties": {},
 589             "name": "testjob"
 590         })
 591
 592         self.assertFalse(api.collections().create.called)
 593         self.assertFalse(runner.runtime_status_error.called)
 594
 595         # Assert that something was written to the usage report
 596         self.assertTrue(len(usage_report.getvalue()) > 0)
 597
 598         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 599         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 600         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 601
 602         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 603                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 604
 605
 606     # Test to make sure we dont call runtime_status_update if we already did
 607     # some where higher up in the call stack
 608     @mock.patch("arvados_cwl.util.get_current_container")
 609     def test_recursive_runtime_status_update(self, gcc_mock):
 610         self.setup_and_test_container_executor_and_logging(gcc_mock)
 611         root_logger = logging.getLogger('')
 612
 613         # get_current_container is invoked when we call runtime_status_update
 614         # so try and log again!
 615         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 616         try:
 617             root_logger.error("First Error")
 618         except RuntimeError:
 619             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 620
 621
 622     # Test to make sure that an exception raised from
 623     # get_current_container doesn't cause the logger to raise an
 624     # exception
 625     @mock.patch("arvados_cwl.util.get_current_container")
 626     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 627         self.setup_and_test_container_executor_and_logging(gcc_mock)
 628         root_logger = logging.getLogger('')
 629
 630         # get_current_container is invoked when we call
 631         # runtime_status_update, it is going to also raise an
 632         # exception.
 633         gcc_mock.side_effect = Exception("Second Error")
 634         try:
 635             root_logger.error("First Error")
 636         except Exception:
 637             self.fail("Exception in logger should not propagate")
 638         self.assertTrue(gcc_mock.called)
 639
 640     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 641     @mock.patch("arvados_cwl.util.get_current_container")
 642     @mock.patch("arvados.collection.CollectionReader")
 643     @mock.patch("arvados.collection.Collection")
 644     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 645         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 646
 647         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 648         self.assertTrue(gcc_mock.called)
 649
 650         runner.num_retries = 0
 651         runner.ignore_docker_for_reuse = False
 652         runner.intermediate_output_ttl = 0
 653         runner.secret_store = cwltool.secrets.SecretStore()
 654         runner.label = mock.MagicMock()
 655         runner.label.return_value = '[container testjob]'
 656
 657         runner.api.containers().get().execute.return_value = {
 658             "state":"Complete",
 659             "output": "abc+123",
 660             "exit_code": 1,
 661             "log": "def+234"
 662         }
 663
 664         col().open.return_value = []
 665
 666         loadingContext, runtimeContext = self.helper(runner)
 667
 668         arvjob = arvados_cwl.ArvadosContainer(runner,
 669                                               runtimeContext,
 670                                               mock.MagicMock(),
 671                                               {},
 672                                               None,
 673                                               [],
 674                                               [],
 675                                               "testjob")
 676         arvjob.output_callback = mock.MagicMock()
 677         arvjob.collect_outputs = mock.MagicMock()
 678         arvjob.successCodes = [0]
 679         arvjob.outdir = "/var/spool/cwl"
 680         arvjob.output_ttl = 3600
 681         arvjob.collect_outputs.return_value = {"out": "stuff"}
 682
 683         arvjob.done({
 684             "state": "Final",
 685             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 686             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 687             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 688             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 689             "modified_at": "2017-05-26T12:01:22Z",
 690             "properties": {}
 691         })
 692
 693         rts_mock.assert_has_calls([
 694             mock.call('error',
 695                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 696                       '  ** log is empty **'
 697                       ),
 698             mock.call('warning',
 699                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 700         )])
 701         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 702
 703     # The test passes no builder.resources
 704     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 705     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 706     def test_mounts(self, keepdocker):
 707         runner = mock.MagicMock()
 708         runner.ignore_docker_for_reuse = False
 709         runner.intermediate_output_ttl = 0
 710         runner.secret_store = cwltool.secrets.SecretStore()
 711         runner.api._rootDesc = {"revision": "20210628"}
 712         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 713
 714         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 715         runner.api.collections().get().execute.return_value = {
 716             "portable_data_hash": "99999999999999999999999999999994+99",
 717             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 718
 719         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 720
 721         tool = cmap({
 722             "inputs": [
 723                 {"id": "p1",
 724                  "type": "Directory"}
 725             ],
 726             "outputs": [],
 727             "baseCommand": "ls",
 728             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 729             "id": "",
 730             "class": "CommandLineTool",
 731             "cwlVersion": "v1.2"
 732         })
 733
 734         loadingContext, runtimeContext = self.helper(runner)
 735         runtimeContext.name = "test_run_mounts"
 736
 737         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 738         arvtool.formatgraph = None
 739         job_order = {
 740             "p1": {
 741                 "class": "Directory",
 742                 "location": "keep:99999999999999999999999999999994+44",
 743                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 744                 "listing": [
 745                     {
 746                         "class": "File",
 747                         "location": "keep:99999999999999999999999999999994+44/file1",
 748                     },
 749                     {
 750                         "class": "File",
 751                         "location": "keep:99999999999999999999999999999994+44/file2",
 752                     }
 753                 ]
 754             }
 755         }
 756         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 757             j.run(runtimeContext)
 758             runner.api.container_requests().create.assert_called_with(
 759                 body=JsonDiffMatcher({
 760                     'environment': {
 761                         'HOME': '/var/spool/cwl',
 762                         'TMPDIR': '/tmp'
 763                     },
 764                     'name': 'test_run_mounts',
 765                     'runtime_constraints': {
 766                         'vcpus': 1,
 767                         'ram': 268435456
 768                     },
 769                     'use_existing': True,
 770                     'priority': 500,
 771                     'mounts': {
 772                         "/keep/99999999999999999999999999999994+44": {
 773                             "kind": "collection",
 774                             "portable_data_hash": "99999999999999999999999999999994+44",
 775                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 776                         },
 777                         '/tmp': {'kind': 'tmp',
 778                                  "capacity": 1073741824 },
 779                         '/var/spool/cwl': {'kind': 'tmp',
 780                                            "capacity": 1073741824 }
 781                     },
 782                     'state': 'Committed',
 783                     'output_name': 'Output from step test_run_mounts',
 784                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 785                     'output_path': '/var/spool/cwl',
 786                     'output_ttl': 0,
 787                     'container_image': '99999999999999999999999999999994+99',
 788                     'command': ['ls', '/var/spool/cwl'],
 789                     'cwd': '/var/spool/cwl',
 790                     'scheduling_parameters': {},
 791                     'properties': {'cwl_input': {
 792                         "p1": {
 793                             "basename": "99999999999999999999999999999994+44",
 794                             "class": "Directory",
 795                             "dirname": "/keep",
 796                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 797                             "listing": [
 798                                 {
 799                                     "basename": "file1",
 800                                     "class": "File",
 801                                     "dirname": "/keep/99999999999999999999999999999994+44",
 802                                     "location": "keep:99999999999999999999999999999994+44/file1",
 803                                     "nameext": "",
 804                                     "nameroot": "file1",
 805                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 806                                     "size": 0
 807                                 },
 808                                 {
 809                                     "basename": "file2",
 810                                     "class": "File",
 811                                     "dirname": "/keep/99999999999999999999999999999994+44",
 812                                     "location": "keep:99999999999999999999999999999994+44/file2",
 813                                     "nameext": "",
 814                                     "nameroot": "file2",
 815                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 816                                     "size": 0
 817                                 }
 818                             ],
 819                             "location": "keep:99999999999999999999999999999994+44",
 820                             "path": "/keep/99999999999999999999999999999994+44"
 821                         }
 822                     }},
 823                     'secret_mounts': {},
 824                     'output_storage_classes': ["default"]
 825                 }))
 826
 827     # The test passes no builder.resources
 828     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 829     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 830     def test_secrets(self, keepdocker):
 831         arvados_cwl.add_arv_hints()
 832         runner = mock.MagicMock()
 833         runner.ignore_docker_for_reuse = False
 834         runner.intermediate_output_ttl = 0
 835         runner.secret_store = cwltool.secrets.SecretStore()
 836         runner.api._rootDesc = {"revision": "20210628"}
 837         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 838
 839         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 840         runner.api.collections().get().execute.return_value = {
 841             "portable_data_hash": "99999999999999999999999999999993+99"}
 842
 843         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 844
 845         tool = cmap({"arguments": ["md5sum", "example.conf"],
 846                      "class": "CommandLineTool",
 847                      "cwlVersion": "v1.2",
 848                      "hints": [
 849                          {
 850                              "class": "http://commonwl.org/cwltool#Secrets",
 851                              "secrets": [
 852                                  "#secret_job.cwl/pw"
 853                              ]
 854                          }
 855                      ],
 856                      "id": "",
 857                      "inputs": [
 858                          {
 859                              "id": "#secret_job.cwl/pw",
 860                              "type": "string"
 861                          }
 862                      ],
 863                      "outputs": [
 864                      ],
 865                      "requirements": [
 866                          {
 867                              "class": "InitialWorkDirRequirement",
 868                              "listing": [
 869                                  {
 870                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 871                                      "entryname": "example.conf"
 872                                  }
 873                              ]
 874                          }
 875                      ]})
 876
 877         loadingContext, runtimeContext = self.helper(runner)
 878         runtimeContext.name = "test_secrets"
 879
 880         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 881         arvtool.formatgraph = None
 882
 883         job_order = {"pw": "blorp"}
 884         runner.secret_store.store(["pw"], job_order)
 885
 886         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 887             j.run(runtimeContext)
 888             runner.api.container_requests().create.assert_called_with(
 889                 body=JsonDiffMatcher({
 890                     'environment': {
 891                         'HOME': '/var/spool/cwl',
 892                         'TMPDIR': '/tmp'
 893                     },
 894                     'name': 'test_secrets',
 895                     'runtime_constraints': {
 896                         'vcpus': 1,
 897                         'ram': 268435456
 898                     },
 899                     'use_existing': True,
 900                     'priority': 500,
 901                     'mounts': {
 902                         '/tmp': {'kind': 'tmp',
 903                                  "capacity": 1073741824
 904                              },
 905                         '/var/spool/cwl': {'kind': 'tmp',
 906                                            "capacity": 1073741824 }
 907                     },
 908                     'state': 'Committed',
 909                     'output_name': 'Output from step test_secrets',
 910                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 911                     'output_path': '/var/spool/cwl',
 912                     'output_ttl': 0,
 913                     'container_image': '99999999999999999999999999999993+99',
 914                     'command': ['md5sum', 'example.conf'],
 915                     'cwd': '/var/spool/cwl',
 916                     'scheduling_parameters': {},
 917                     'properties': {'cwl_input': job_order},
 918                     "secret_mounts": {
 919                         "/var/spool/cwl/example.conf": {
 920                             "content": "username: user\npassword: blorp\n",
 921                             "kind": "text"
 922                         }
 923                     },
 924                     'output_storage_classes': ["default"]
 925                 }))
 926
 927     # The test passes no builder.resources
 928     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 929     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 930     def test_timelimit(self, keepdocker):
 931         runner = mock.MagicMock()
 932         runner.ignore_docker_for_reuse = False
 933         runner.intermediate_output_ttl = 0
 934         runner.secret_store = cwltool.secrets.SecretStore()
 935         runner.api._rootDesc = {"revision": "20210628"}
 936         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 937
 938         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 939         runner.api.collections().get().execute.return_value = {
 940             "portable_data_hash": "99999999999999999999999999999993+99"}
 941
 942         tool = cmap({
 943             "inputs": [],
 944             "outputs": [],
 945             "baseCommand": "ls",
 946             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 947             "id": "",
 948             "cwlVersion": "v1.2",
 949             "class": "CommandLineTool",
 950             "hints": [
 951                 {
 952                     "class": "ToolTimeLimit",
 953                     "timelimit": 42
 954                 }
 955             ]
 956         })
 957
 958         loadingContext, runtimeContext = self.helper(runner)
 959         runtimeContext.name = "test_timelimit"
 960
 961         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 962         arvtool.formatgraph = None
 963
 964         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 965             j.run(runtimeContext)
 966
 967         _, kwargs = runner.api.container_requests().create.call_args
 968         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 969
 970
 971     # The test passes no builder.resources
 972     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 973     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 974     def test_setting_storage_class(self, keepdocker):
 975         #arv_docker_clear_cache()
 976
 977         runner = mock.MagicMock()
 978         runner.ignore_docker_for_reuse = False
 979         runner.intermediate_output_ttl = 0
 980         runner.secret_store = cwltool.secrets.SecretStore()
 981         runner.api._rootDesc = {"revision": "20210628"}
 982         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 983
 984         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 985         runner.api.collections().get().execute.return_value = {
 986             "portable_data_hash": "99999999999999999999999999999993+99"}
 987
 988         tool = cmap({
 989             "inputs": [],
 990             "outputs": [],
 991             "baseCommand": "ls",
 992             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 993             "id": "",
 994             "cwlVersion": "v1.2",
 995             "class": "CommandLineTool",
 996             "hints": [
 997                 {
 998                     "class": "http://arvados.org/cwl#OutputStorageClass",
 999                     "finalStorageClass": ["baz_sc", "qux_sc"],
1000                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1001                 }
1002             ]
1003         })
1004
1005         loadingContext, runtimeContext = self.helper(runner, True)
1006
1007         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1008         arvtool.formatgraph = None
1009
1010         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1011             j.run(runtimeContext)
1012             runner.api.container_requests().create.assert_called_with(
1013                 body=JsonDiffMatcher({
1014                     'environment': {
1015                         'HOME': '/var/spool/cwl',
1016                         'TMPDIR': '/tmp'
1017                     },
1018                     'name': 'test_run_True',
1019                     'runtime_constraints': {
1020                         'vcpus': 1,
1021                         'ram': 268435456
1022                     },
1023                     'use_existing': True,
1024                     'priority': 500,
1025                     'mounts': {
1026                         '/tmp': {'kind': 'tmp',
1027                                  "capacity": 1073741824
1028                              },
1029                         '/var/spool/cwl': {'kind': 'tmp',
1030                                            "capacity": 1073741824 }
1031                     },
1032                     'state': 'Committed',
1033                     'output_name': 'Output from step test_run_True',
1034                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1035                     'output_path': '/var/spool/cwl',
1036                     'output_ttl': 0,
1037                     'container_image': '99999999999999999999999999999993+99',
1038                     'command': ['ls', '/var/spool/cwl'],
1039                     'cwd': '/var/spool/cwl',
1040                     'scheduling_parameters': {},
1041                     'properties': {'cwl_input': {}},
1042                     'secret_mounts': {},
1043                     'output_storage_classes': ["foo_sc", "bar_sc"]
1044                 }))
1045
1046
1047     # The test passes no builder.resources
1048     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1049     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1050     def test_setting_process_properties(self, keepdocker):
1051         #arv_docker_clear_cache()
1052
1053         runner = mock.MagicMock()
1054         runner.ignore_docker_for_reuse = False
1055         runner.intermediate_output_ttl = 0
1056         runner.secret_store = cwltool.secrets.SecretStore()
1057         runner.api._rootDesc = {"revision": "20210628"}
1058         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1059
1060         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1061         runner.api.collections().get().execute.return_value = {
1062             "portable_data_hash": "99999999999999999999999999999993+99"}
1063
1064         tool = cmap({
1065             "inputs": [
1066                 {"id": "x", "type": "string"}],
1067             "outputs": [],
1068             "baseCommand": "ls",
1069             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1070             "id": "",
1071             "class": "CommandLineTool",
1072             "cwlVersion": "v1.2",
1073             "hints": [
1074             {
1075                 "class": "http://arvados.org/cwl#ProcessProperties",
1076                 "processProperties": [
1077                     {"propertyName": "foo",
1078                      "propertyValue": "bar"},
1079                     {"propertyName": "baz",
1080                      "propertyValue": "$(inputs.x)"},
1081                     {"propertyName": "quux",
1082                      "propertyValue": {
1083                          "q1": 1,
1084                          "q2": 2
1085                      }
1086                     }
1087                 ],
1088             }
1089         ]
1090         })
1091
1092         loadingContext, runtimeContext = self.helper(runner, True)
1093
1094         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1095         arvtool.formatgraph = None
1096
1097         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1098             j.run(runtimeContext)
1099             runner.api.container_requests().create.assert_called_with(
1100                 body=JsonDiffMatcher({
1101                     'environment': {
1102                         'HOME': '/var/spool/cwl',
1103                         'TMPDIR': '/tmp'
1104                     },
1105                     'name': 'test_run_True',
1106                     'runtime_constraints': {
1107                         'vcpus': 1,
1108                         'ram': 268435456
1109                     },
1110                     'use_existing': True,
1111                     'priority': 500,
1112                     'mounts': {
1113                         '/tmp': {'kind': 'tmp',
1114                                  "capacity": 1073741824
1115                              },
1116                         '/var/spool/cwl': {'kind': 'tmp',
1117                                            "capacity": 1073741824 }
1118                     },
1119                     'state': 'Committed',
1120                     'output_name': 'Output from step test_run_True',
1121                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1122                     'output_path': '/var/spool/cwl',
1123                     'output_ttl': 0,
1124                     'container_image': '99999999999999999999999999999993+99',
1125                     'command': ['ls', '/var/spool/cwl'],
1126                     'cwd': '/var/spool/cwl',
1127                     'scheduling_parameters': {},
1128                     'properties': {
1129                         "baz": "blorp",
1130                         "cwl_input": {"x": "blorp"},
1131                         "foo": "bar",
1132                         "quux": {
1133                             "q1": 1,
1134                             "q2": 2
1135                         }
1136                     },
1137                     'secret_mounts': {},
1138                     'output_storage_classes': ["default"]
1139                 }))
1140
1141
1142     # The test passes no builder.resources
1143     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1144     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1145     def test_cuda_requirement(self, keepdocker):
1146         arvados_cwl.add_arv_hints()
1147         #arv_docker_clear_cache()
1148
1149         runner = mock.MagicMock()
1150         runner.ignore_docker_for_reuse = False
1151         runner.intermediate_output_ttl = 0
1152         runner.secret_store = cwltool.secrets.SecretStore()
1153         runner.api._rootDesc = {"revision": "20210628"}
1154         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1155
1156         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1157         runner.api.collections().get().execute.return_value = {
1158             "portable_data_hash": "99999999999999999999999999999993+99"}
1159
1160         test_cwl_req = [{
1161                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1162                 "cudaVersionMin": "11.0",
1163                 "cudaComputeCapability": "9.0",
1164             }, {
1165                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1166                 "cudaVersionMin": "11.0",
1167                 "cudaComputeCapability": "9.0",
1168                 "cudaDeviceCountMin": 2
1169             }, {
1170                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1171                 "cudaVersionMin": "11.0",
1172                 "cudaComputeCapability": ["4.0", "5.0"],
1173                 "cudaDeviceCountMin": 2
1174             }]
1175
1176         test_arv_req = [{
1177             'device_count': 1,
1178             'driver_version': "11.0",
1179             'hardware_capability': "9.0"
1180         }, {
1181             'device_count': 2,
1182             'driver_version': "11.0",
1183             'hardware_capability': "9.0"
1184         }, {
1185             'device_count': 2,
1186             'driver_version': "11.0",
1187             'hardware_capability': "4.0"
1188         }]
1189
1190         for test_case in range(0, len(test_cwl_req)):
1191
1192             tool = cmap({
1193                 "inputs": [],
1194                 "outputs": [],
1195                 "baseCommand": "nvidia-smi",
1196                 "arguments": [],
1197                 "id": "",
1198                 "cwlVersion": "v1.2",
1199                 "class": "CommandLineTool",
1200                 "requirements": [test_cwl_req[test_case]]
1201             })
1202
1203             loadingContext, runtimeContext = self.helper(runner, True)
1204
1205             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1206             arvtool.formatgraph = None
1207
1208             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1209                 j.run(runtimeContext)
1210                 runner.api.container_requests().create.assert_called_with(
1211                     body=JsonDiffMatcher({
1212                         'environment': {
1213                             'HOME': '/var/spool/cwl',
1214                             'TMPDIR': '/tmp'
1215                         },
1216                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1217                         'runtime_constraints': {
1218                             'vcpus': 1,
1219                             'ram': 268435456,
1220                             'cuda': test_arv_req[test_case]
1221                         },
1222                         'use_existing': True,
1223                         'priority': 500,
1224                         'mounts': {
1225                             '/tmp': {'kind': 'tmp',
1226                                      "capacity": 1073741824
1227                                  },
1228                             '/var/spool/cwl': {'kind': 'tmp',
1229                                                "capacity": 1073741824 }
1230                         },
1231                         'state': 'Committed',
1232                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1233                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1234                         'output_path': '/var/spool/cwl',
1235                         'output_ttl': 0,
1236                         'container_image': '99999999999999999999999999999993+99',
1237                         'command': ['nvidia-smi'],
1238                         'cwd': '/var/spool/cwl',
1239                         'scheduling_parameters': {},
1240                         'properties': {'cwl_input': {}},
1241                         'secret_mounts': {},
1242                         'output_storage_classes': ["default"]
1243                     }))
1244
1245
1246     # The test passes no builder.resources
1247     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1248     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1249     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1250     def test_match_local_docker(self, keepdocker, determine_image_id):
1251         arvados_cwl.add_arv_hints()
1252
1253         runner = mock.MagicMock()
1254         runner.ignore_docker_for_reuse = False
1255         runner.intermediate_output_ttl = 0
1256         runner.secret_store = cwltool.secrets.SecretStore()
1257         runner.api._rootDesc = {"revision": "20210628"}
1258         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1259
1260         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1261                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1262         determine_image_id.side_effect = lambda x: "123"
1263         def execute(uuid):
1264             ex = mock.MagicMock()
1265             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1266                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1267             ex.execute.return_value = lookup[uuid]
1268             return ex
1269         runner.api.collections().get.side_effect = execute
1270
1271         tool = cmap({
1272             "inputs": [],
1273             "outputs": [],
1274             "baseCommand": "echo",
1275             "arguments": [],
1276             "id": "",
1277             "cwlVersion": "v1.0",
1278             "class": "org.w3id.cwl.cwl.CommandLineTool"
1279         })
1280
1281         loadingContext, runtimeContext = self.helper(runner, True)
1282
1283         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1284         arvtool.formatgraph = None
1285
1286         container_request = {
1287             'environment': {
1288                 'HOME': '/var/spool/cwl',
1289                 'TMPDIR': '/tmp'
1290             },
1291             'name': 'test_run_True',
1292             'runtime_constraints': {
1293                 'vcpus': 1,
1294                 'ram': 1073741824,
1295             },
1296             'use_existing': True,
1297             'priority': 500,
1298             'mounts': {
1299                 '/tmp': {'kind': 'tmp',
1300                          "capacity": 1073741824
1301                          },
1302                 '/var/spool/cwl': {'kind': 'tmp',
1303                                    "capacity": 1073741824 }
1304             },
1305             'state': 'Committed',
1306             'output_name': 'Output from step test_run_True',
1307             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1308             'output_path': '/var/spool/cwl',
1309             'output_ttl': 0,
1310             'container_image': '99999999999999999999999999999994+99',
1311             'command': ['echo'],
1312             'cwd': '/var/spool/cwl',
1313             'scheduling_parameters': {},
1314             'properties': {'cwl_input': {}},
1315             'secret_mounts': {},
1316             'output_storage_classes': ["default"]
1317         }
1318
1319         runtimeContext.match_local_docker = False
1320         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1321             j.run(runtimeContext)
1322             runner.api.container_requests().create.assert_called_with(
1323                 body=JsonDiffMatcher(container_request))
1324
1325         runtimeContext.cached_docker_lookups.clear()
1326         runtimeContext.match_local_docker = True
1327         container_request['container_image'] = '99999999999999999999999999999993+99'
1328         container_request['name'] = 'test_run_True_2'
1329         container_request['output_name'] = 'Output from step test_run_True_2'
1330         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1331             j.run(runtimeContext)
1332             runner.api.container_requests().create.assert_called_with(
1333                 body=JsonDiffMatcher(container_request))
1334
1335
1336     # The test passes no builder.resources
1337     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1338     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1339     def test_run_preemptible_hint(self, keepdocker):
1340         arvados_cwl.add_arv_hints()
1341         for enable_preemptible in (None, True, False):
1342             for preemptible_hint in (None, True, False):
1343                 #arv_docker_clear_cache()
1344
1345                 runner = mock.MagicMock()
1346                 runner.ignore_docker_for_reuse = False
1347                 runner.intermediate_output_ttl = 0
1348                 runner.secret_store = cwltool.secrets.SecretStore()
1349                 runner.api._rootDesc = {"revision": "20210628"}
1350                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1351
1352                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1353                 runner.api.collections().get().execute.return_value = {
1354                     "portable_data_hash": "99999999999999999999999999999993+99"}
1355
1356                 if preemptible_hint is not None:
1357                     hints = [{
1358                         "class": "http://arvados.org/cwl#UsePreemptible",
1359                         "usePreemptible": preemptible_hint
1360                     }]
1361                 else:
1362                     hints = []
1363
1364                 tool = cmap({
1365                     "inputs": [],
1366                     "outputs": [],
1367                     "baseCommand": "ls",
1368                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1369                     "id": "",
1370                     "class": "CommandLineTool",
1371                     "cwlVersion": "v1.2",
1372                     "hints": hints
1373                 })
1374
1375                 loadingContext, runtimeContext = self.helper(runner)
1376
1377                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1378                 runtimeContext.enable_preemptible = enable_preemptible
1379
1380                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1381                 arvtool.formatgraph = None
1382
1383                 # Test the interactions between --enable/disable-preemptible
1384                 # and UsePreemptible hint
1385
1386                 if enable_preemptible is None:
1387                     if preemptible_hint is None:
1388                         sched = {}
1389                     else:
1390                         sched = {'preemptible': preemptible_hint}
1391                 else:
1392                     if preemptible_hint is None:
1393                         sched = {'preemptible': enable_preemptible}
1394                     else:
1395                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1396
1397                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1398                     j.run(runtimeContext)
1399                     runner.api.container_requests().create.assert_called_with(
1400                         body=JsonDiffMatcher({
1401                             'environment': {
1402                                 'HOME': '/var/spool/cwl',
1403                                 'TMPDIR': '/tmp'
1404                             },
1405                             'name': runtimeContext.name,
1406                             'runtime_constraints': {
1407                                 'vcpus': 1,
1408                                 'ram': 268435456
1409                             },
1410                             'use_existing': True,
1411                             'priority': 500,
1412                             'mounts': {
1413                                 '/tmp': {'kind': 'tmp',
1414                                          "capacity": 1073741824
1415                                      },
1416                                 '/var/spool/cwl': {'kind': 'tmp',
1417                                                    "capacity": 1073741824 }
1418                             },
1419                             'state': 'Committed',
1420                             'output_name': 'Output from step '+runtimeContext.name,
1421                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1422                             'output_path': '/var/spool/cwl',
1423                             'output_ttl': 0,
1424                             'container_image': '99999999999999999999999999999993+99',
1425                             'command': ['ls', '/var/spool/cwl'],
1426                             'cwd': '/var/spool/cwl',
1427                             'scheduling_parameters': sched,
1428                             'properties': {'cwl_input': {}},
1429                             'secret_mounts': {},
1430                             'output_storage_classes': ["default"]
1431                         }))
1432
1433
1434     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1435     def test_output_properties(self, keepdocker):
1436         arvados_cwl.add_arv_hints()
1437         for rev in ["20210628", "20220510"]:
1438             runner = mock.MagicMock()
1439             runner.ignore_docker_for_reuse = False
1440             runner.intermediate_output_ttl = 0
1441             runner.secret_store = cwltool.secrets.SecretStore()
1442             runner.api._rootDesc = {"revision": rev}
1443             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1444
1445             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1446             runner.api.collections().get().execute.return_value = {
1447                 "portable_data_hash": "99999999999999999999999999999993+99"}
1448
1449             tool = cmap({
1450                 "inputs": [{
1451                     "id": "inp",
1452                     "type": "string"
1453                 }],
1454                 "outputs": [],
1455                 "baseCommand": "ls",
1456                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1457                 "id": "",
1458                 "cwlVersion": "v1.2",
1459                 "class": "CommandLineTool",
1460                 "hints": [
1461                     {
1462                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1463                         "outputProperties": {
1464                             "foo": "bar",
1465                             "baz": "$(inputs.inp)"
1466                         }
1467                     }
1468                 ]
1469             })
1470
1471             loadingContext, runtimeContext = self.helper(runner)
1472             runtimeContext.name = "test_timelimit"
1473
1474             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1475             arvtool.formatgraph = None
1476
1477             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1478                 j.run(runtimeContext)
1479
1480             _, kwargs = runner.api.container_requests().create.call_args
1481             if rev == "20220510":
1482                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1483             else:
1484                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1485
1486
1487 class TestWorkflow(unittest.TestCase):
1488     def setUp(self):
1489         cwltool.process._names = set()
1490         #arv_docker_clear_cache()
1491
1492     def helper(self, runner, enable_reuse=True):
1493         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1494
1495         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1496                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1497
1498         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1499         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1500         document_loader.fetch_text = document_loader.fetcher.fetch_text
1501         document_loader.check_exists = document_loader.fetcher.check_exists
1502
1503         loadingContext = arvados_cwl.context.ArvLoadingContext(
1504             {"avsc_names": avsc_names,
1505              "basedir": "",
1506              "make_fs_access": make_fs_access,
1507              "loader": document_loader,
1508              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1509              "construct_tool_object": runner.arv_make_tool,
1510              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
1511         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1512             {"work_api": "containers",
1513              "basedir": "",
1514              "name": "test_run_wf_"+str(enable_reuse),
1515              "make_fs_access": make_fs_access,
1516              "tmpdir": "/tmp",
1517              "enable_reuse": enable_reuse,
1518              "priority": 500})
1519
1520         return loadingContext, runtimeContext
1521
1522     # The test passes no builder.resources
1523     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1524     @mock.patch("arvados.collection.CollectionReader")
1525     @mock.patch("arvados.collection.Collection")
1526     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1527     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1528         arvados_cwl.add_arv_hints()
1529
1530         api = mock.MagicMock()
1531         api._rootDesc = get_rootDesc()
1532         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1533
1534         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1535         self.assertEqual(runner.work_api, 'containers')
1536
1537         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1538         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1539         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1540                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1541
1542         runner.api.containers().current().execute.return_value = {}
1543
1544         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1545         runner.ignore_docker_for_reuse = False
1546         runner.num_retries = 0
1547         runner.secret_store = cwltool.secrets.SecretStore()
1548
1549         loadingContext, runtimeContext = self.helper(runner)
1550         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1551
1552         mockcollectionreader().exists.return_value = True
1553
1554         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1555         metadata["cwlVersion"] = tool["cwlVersion"]
1556
1557         mockc = mock.MagicMock()
1558         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1559         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1560
1561         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1562         arvtool.formatgraph = None
1563         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1564
1565         next(it).run(runtimeContext)
1566         next(it).run(runtimeContext)
1567
1568         with open("tests/wf/scatter2_subwf.cwl") as f:
1569             subwf = StripYAMLComments(f.read()).rstrip()
1570
1571         runner.api.container_requests().create.assert_called_with(
1572             body=JsonDiffMatcher({
1573                 "command": [
1574                     "cwltool",
1575                     "--no-container",
1576                     "--move-outputs",
1577                     "--preserve-entire-environment",
1578                     "workflow.cwl",
1579                     "cwl.input.yml"
1580                 ],
1581                 "container_image": "99999999999999999999999999999993+99",
1582                 "cwd": "/var/spool/cwl",
1583                 "environment": {
1584                     "HOME": "/var/spool/cwl",
1585                     "TMPDIR": "/tmp"
1586                 },
1587                 "mounts": {
1588                     "/keep/99999999999999999999999999999999+118": {
1589                         "kind": "collection",
1590                         "portable_data_hash": "99999999999999999999999999999999+118"
1591                     },
1592                     "/tmp": {
1593                         "capacity": 1073741824,
1594                         "kind": "tmp"
1595                     },
1596                     "/var/spool/cwl": {
1597                         "capacity": 1073741824,
1598                         "kind": "tmp"
1599                     },
1600                     "/var/spool/cwl/cwl.input.yml": {
1601                         "kind": "collection",
1602                         "path": "cwl.input.yml",
1603                         "portable_data_hash": "99999999999999999999999999999996+99"
1604                     },
1605                     "/var/spool/cwl/workflow.cwl": {
1606                         "kind": "collection",
1607                         "path": "workflow.cwl",
1608                         "portable_data_hash": "99999999999999999999999999999996+99"
1609                     },
1610                     "stdout": {
1611                         "kind": "file",
1612                         "path": "/var/spool/cwl/cwl.output.json"
1613                     }
1614                 },
1615                 "name": "scatterstep",
1616                 "output_name": "Output from step scatterstep",
1617                 "output_path": "/var/spool/cwl",
1618                 "output_ttl": 0,
1619                 "priority": 500,
1620                 "properties": {'cwl_input': {
1621                         "fileblub": {
1622                             "basename": "token.txt",
1623                             "class": "File",
1624                             "dirname": "/keep/99999999999999999999999999999999+118",
1625                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1626                             "nameext": ".txt",
1627                             "nameroot": "token",
1628                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1629                             "size": 0
1630                         },
1631                         "sleeptime": 5
1632                 }},
1633                 "runtime_constraints": {
1634                     "ram": 1073741824,
1635                     "vcpus": 1
1636                 },
1637                 "scheduling_parameters": {},
1638                 "secret_mounts": {},
1639                 "state": "Committed",
1640                 "use_existing": True,
1641                 'output_storage_classes': ["default"]
1642             }))
1643         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1644         mockc.open().__enter__().write.assert_has_calls([mock.call(
1645 '''{
1646   "fileblub": {
1647     "basename": "token.txt",
1648     "class": "File",
1649     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1650     "size": 0
1651   },
1652   "sleeptime": 5
1653 }''')])
1654
1655     # The test passes no builder.resources
1656     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1657     @mock.patch("arvados.collection.CollectionReader")
1658     @mock.patch("arvados.collection.Collection")
1659     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1660     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1661         arvados_cwl.add_arv_hints()
1662
1663         api = mock.MagicMock()
1664         api._rootDesc = get_rootDesc()
1665         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1666
1667         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1668         self.assertEqual(runner.work_api, 'containers')
1669
1670         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1671         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1672                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1673         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1674                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1675
1676         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1677         runner.ignore_docker_for_reuse = False
1678         runner.num_retries = 0
1679         runner.secret_store = cwltool.secrets.SecretStore()
1680
1681         loadingContext, runtimeContext = self.helper(runner)
1682         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1683         loadingContext.do_update = True
1684         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1685
1686         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1687
1688         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1689         arvtool.formatgraph = None
1690         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1691
1692         next(it).run(runtimeContext)
1693         next(it).run(runtimeContext)
1694
1695         with open("tests/wf/echo-subwf.cwl") as f:
1696             subwf = StripYAMLComments(f.read())
1697
1698         runner.api.container_requests().create.assert_called_with(
1699             body=JsonDiffMatcher({
1700                 'output_ttl': 0,
1701                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1702                 'scheduling_parameters': {},
1703                 'name': u'echo-subwf',
1704                 'secret_mounts': {},
1705                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1706                 'properties': {'cwl_input': {}},
1707                 'priority': 500,
1708                 'mounts': {
1709                     '/var/spool/cwl/cwl.input.yml': {
1710                         'portable_data_hash': '99999999999999999999999999999996+99',
1711                         'kind': 'collection',
1712                         'path': 'cwl.input.yml'
1713                     },
1714                     '/var/spool/cwl/workflow.cwl': {
1715                         'portable_data_hash': '99999999999999999999999999999996+99',
1716                         'kind': 'collection',
1717                         'path': 'workflow.cwl'
1718                     },
1719                     'stdout': {
1720                         'path': '/var/spool/cwl/cwl.output.json',
1721                         'kind': 'file'
1722                     },
1723                     '/tmp': {
1724                         'kind': 'tmp',
1725                         'capacity': 1073741824
1726                     }, '/var/spool/cwl': {
1727                         'kind': 'tmp',
1728                         'capacity': 3221225472
1729                     }
1730                 },
1731                 'state': 'Committed',
1732                 'output_path': '/var/spool/cwl',
1733                 'container_image': '99999999999999999999999999999993+99',
1734                 'command': [
1735                     u'cwltool',
1736                     u'--no-container',
1737                     u'--move-outputs',
1738                     u'--preserve-entire-environment',
1739                     u'workflow.cwl',
1740                     u'cwl.input.yml'
1741                 ],
1742                 'use_existing': True,
1743                 'output_name': u'Output from step echo-subwf',
1744                 'cwd': '/var/spool/cwl',
1745                 'output_storage_classes': ["default"]
1746             }))
1747
1748     def test_default_work_api(self):
1749         arvados_cwl.add_arv_hints()
1750
1751         api = mock.MagicMock()
1752         api._rootDesc = copy.deepcopy(get_rootDesc())
1753         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1754         self.assertEqual(runner.work_api, 'containers')