sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 #from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26 import io
  27
  28 from .matcher import JsonDiffMatcher, StripYAMLComments
  29 from .mock_discovery import get_rootDesc
  30
  31 if not os.getenv('ARVADOS_DEBUG'):
  32     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  33     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  34
  35 class CollectionMock(object):
  36     def __init__(self, vwdmock, *args, **kwargs):
  37         self.vwdmock = vwdmock
  38         self.count = 0
  39
  40     def open(self, *args, **kwargs):
  41         self.count += 1
  42         return self.vwdmock.open(*args, **kwargs)
  43
  44     def copy(self, *args, **kwargs):
  45         self.count += 1
  46         self.vwdmock.copy(*args, **kwargs)
  47
  48     def save_new(self, *args, **kwargs):
  49         pass
  50
  51     def __len__(self):
  52         return self.count
  53
  54     def portable_data_hash(self):
  55         if self.count == 0:
  56             return arvados.config.EMPTY_BLOCK_LOCATOR
  57         else:
  58             return "99999999999999999999999999999996+99"
  59
  60
  61 class TestContainer(unittest.TestCase):
  62
  63     def setUp(self):
  64         cwltool.process._names = set()
  65         #arv_docker_clear_cache()
  66
  67     def tearDown(self):
  68         root_logger = logging.getLogger('')
  69
  70         # Remove existing RuntimeStatusLoggingHandlers if they exist
  71         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  72         root_logger.handlers = handlers
  73
  74     def helper(self, runner, enable_reuse=True):
  75         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  76
  77         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  78                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  79         fs_access = mock.MagicMock()
  80         fs_access.exists.return_value = True
  81
  82         loadingContext = arvados_cwl.context.ArvLoadingContext(
  83             {"avsc_names": avsc_names,
  84              "basedir": "",
  85              "make_fs_access": make_fs_access,
  86              "construct_tool_object": runner.arv_make_tool,
  87              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  88              "loader": Loader({}),
  89              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  90              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
  91              })
  92         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  93             {"work_api": "containers",
  94              "basedir": "",
  95              "name": "test_run_"+str(enable_reuse),
  96              "make_fs_access": make_fs_access,
  97              "tmpdir": "/tmp",
  98              "outdir": "/tmp",
  99              "enable_reuse": enable_reuse,
 100              "priority": 500,
 101              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 102              "workflow_eval_lock": threading.Condition(threading.RLock())
 103             })
 104
 105         if isinstance(runner, mock.MagicMock):
 106             def make_tool(toolpath_object, loadingContext):
 107                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 108             runner.arv_make_tool.side_effect = make_tool
 109
 110         return loadingContext, runtimeContext
 111
 112     # Helper function to set up the ArvCwlExecutor to use the containers api
 113     # and test that the RuntimeStatusLoggingHandler is set up correctly
 114     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 115         api = mock.MagicMock()
 116         api._rootDesc = copy.deepcopy(get_rootDesc())
 117
 118         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 119         # adds the logging handler that will call runtime_status_update() mock
 120         self.assertFalse(gcc_mock.called)
 121         runner = arvados_cwl.ArvCwlExecutor(api)
 122         self.assertEqual(runner.work_api, 'containers')
 123         root_logger = logging.getLogger('')
 124         handlerClasses = [h.__class__ for h in root_logger.handlers]
 125         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 126         return runner
 127
 128     # The test passes no builder.resources
 129     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 130     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 131     def test_run(self, keepdocker):
 132         for enable_reuse in (True, False):
 133             #arv_docker_clear_cache()
 134
 135             runner = mock.MagicMock()
 136             runner.ignore_docker_for_reuse = False
 137             runner.intermediate_output_ttl = 0
 138             runner.secret_store = cwltool.secrets.SecretStore()
 139             runner.api._rootDesc = {"revision": "20210628"}
 140             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 141
 142             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 143             runner.api.collections().get().execute.return_value = {
 144                 "portable_data_hash": "99999999999999999999999999999993+99"}
 145
 146             tool = cmap({
 147                 "inputs": [],
 148                 "outputs": [],
 149                 "baseCommand": "ls",
 150                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 151                 "id": "",
 152                 "class": "CommandLineTool",
 153                 "cwlVersion": "v1.2"
 154             })
 155
 156             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 157
 158             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 159             arvtool.formatgraph = None
 160
 161             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 162                 j.run(runtimeContext)
 163                 runner.api.container_requests().create.assert_called_with(
 164                     body=JsonDiffMatcher({
 165                         'environment': {
 166                             'HOME': '/var/spool/cwl',
 167                             'TMPDIR': '/tmp'
 168                         },
 169                         'name': 'test_run_'+str(enable_reuse),
 170                         'runtime_constraints': {
 171                             'vcpus': 1,
 172                             'ram': 268435456
 173                         },
 174                         'use_existing': enable_reuse,
 175                         'priority': 500,
 176                         'mounts': {
 177                             '/tmp': {'kind': 'tmp',
 178                                      "capacity": 1073741824
 179                                  },
 180                             '/var/spool/cwl': {'kind': 'tmp',
 181                                                "capacity": 1073741824 }
 182                         },
 183                         'state': 'Committed',
 184                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 185                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 186                         'output_path': '/var/spool/cwl',
 187                         'output_ttl': 0,
 188                         'container_image': '99999999999999999999999999999993+99',
 189                         'command': ['ls', '/var/spool/cwl'],
 190                         'cwd': '/var/spool/cwl',
 191                         'scheduling_parameters': {},
 192                         'properties': {'cwl_input': {}},
 193                         'secret_mounts': {},
 194                         'output_storage_classes': ["default"]
 195                     }))
 196
 197     # The test passes some fields in builder.resources
 198     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 199     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 200     def test_resource_requirements(self, keepdocker):
 201         arvados_cwl.add_arv_hints()
 202         runner = mock.MagicMock()
 203         runner.ignore_docker_for_reuse = False
 204         runner.intermediate_output_ttl = 3600
 205         runner.secret_store = cwltool.secrets.SecretStore()
 206         runner.api._rootDesc = {"revision": "20210628"}
 207         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 208
 209         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 210         runner.api.collections().get().execute.return_value = {
 211             "portable_data_hash": "99999999999999999999999999999993+99"}
 212
 213         tool = cmap({
 214             "inputs": [],
 215             "outputs": [],
 216             "hints": [{
 217                 "class": "ResourceRequirement",
 218                 "coresMin": 3,
 219                 "ramMin": 3000,
 220                 "tmpdirMin": 4000,
 221                 "outdirMin": 5000
 222             }, {
 223                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 224                 "keep_cache": 512
 225             }, {
 226                 "class": "http://arvados.org/cwl#APIRequirement",
 227             }, {
 228                 "class": "http://arvados.org/cwl#PartitionRequirement",
 229                 "partition": "blurb"
 230             }, {
 231                 "class": "http://arvados.org/cwl#IntermediateOutput",
 232                 "outputTTL": 7200
 233             }, {
 234                 "class": "WorkReuse",
 235                 "enableReuse": False
 236             }],
 237             "baseCommand": "ls",
 238             "id": "",
 239             "class": "CommandLineTool",
 240             "cwlVersion": "v1.2"
 241         })
 242
 243         loadingContext, runtimeContext = self.helper(runner)
 244         runtimeContext.name = "test_resource_requirements"
 245
 246         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 247         arvtool.formatgraph = None
 248         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 249             j.run(runtimeContext)
 250
 251         call_args, call_kwargs = runner.api.container_requests().create.call_args
 252
 253         call_body_expected = {
 254             'environment': {
 255                 'HOME': '/var/spool/cwl',
 256                 'TMPDIR': '/tmp'
 257             },
 258             'name': 'test_resource_requirements',
 259             'runtime_constraints': {
 260                 'vcpus': 3,
 261                 'ram': 3145728000,
 262                 'keep_cache_ram': 536870912,
 263                 'API': True
 264             },
 265             'use_existing': False,
 266             'priority': 500,
 267             'mounts': {
 268                 '/tmp': {'kind': 'tmp',
 269                          "capacity": 4194304000 },
 270                 '/var/spool/cwl': {'kind': 'tmp',
 271                                    "capacity": 5242880000 }
 272             },
 273             'state': 'Committed',
 274             'output_name': 'Output from step test_resource_requirements',
 275             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 276             'output_path': '/var/spool/cwl',
 277             'output_ttl': 7200,
 278             'container_image': '99999999999999999999999999999993+99',
 279             'command': ['ls'],
 280             'cwd': '/var/spool/cwl',
 281             'scheduling_parameters': {
 282                 'partitions': ['blurb']
 283             },
 284             'properties': {'cwl_input': {}},
 285             'secret_mounts': {},
 286             'output_storage_classes': ["default"]
 287         }
 288
 289         call_body = call_kwargs.get('body', None)
 290         self.assertNotEqual(None, call_body)
 291         for key in call_body:
 292             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 293
 294
 295     # The test passes some fields in builder.resources
 296     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 297     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 298     @mock.patch("arvados.collection.Collection")
 299     def test_initial_work_dir(self, collection_mock, keepdocker):
 300         runner = mock.MagicMock()
 301         runner.ignore_docker_for_reuse = False
 302         runner.intermediate_output_ttl = 0
 303         runner.secret_store = cwltool.secrets.SecretStore()
 304         runner.api._rootDesc = {"revision": "20210628"}
 305         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 306
 307         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 308         runner.api.collections().get().execute.return_value = {
 309             "portable_data_hash": "99999999999999999999999999999993+99"}
 310
 311         sourcemock = mock.MagicMock()
 312         def get_collection_mock(p):
 313             if "/" in p:
 314                 return (sourcemock, p.split("/", 1)[1])
 315             else:
 316                 return (sourcemock, "")
 317         runner.fs_access.get_collection.side_effect = get_collection_mock
 318
 319         vwdmock = mock.MagicMock()
 320         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 321
 322         tool = cmap({
 323             "inputs": [],
 324             "outputs": [],
 325             "hints": [{
 326                 "class": "InitialWorkDirRequirement",
 327                 "listing": [{
 328                     "class": "File",
 329                     "basename": "foo",
 330                     "location": "keep:99999999999999999999999999999995+99/bar"
 331                 },
 332                 {
 333                     "class": "Directory",
 334                     "basename": "foo2",
 335                     "location": "keep:99999999999999999999999999999995+99"
 336                 },
 337                 {
 338                     "class": "File",
 339                     "basename": "filename",
 340                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 341                 },
 342                 {
 343                     "class": "Directory",
 344                     "basename": "subdir",
 345                     "location": "keep:99999999999999999999999999999995+99/subdir"
 346                 }                        ]
 347             }],
 348             "baseCommand": "ls",
 349             "class": "CommandLineTool",
 350             "cwlVersion": "v1.2",
 351             "id": ""
 352         })
 353
 354         loadingContext, runtimeContext = self.helper(runner)
 355         runtimeContext.name = "test_initial_work_dir"
 356
 357         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 358
 359         arvtool.formatgraph = None
 360         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 361             j.run(runtimeContext)
 362
 363         call_args, call_kwargs = runner.api.container_requests().create.call_args
 364
 365         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 367         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 368         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 369
 370         call_body_expected = {
 371             'environment': {
 372                 'HOME': '/var/spool/cwl',
 373                 'TMPDIR': '/tmp'
 374             },
 375             'name': 'test_initial_work_dir',
 376             'runtime_constraints': {
 377                 'vcpus': 1,
 378                 'ram': 268435456
 379             },
 380             'use_existing': True,
 381             'priority': 500,
 382             'mounts': {
 383                 '/tmp': {'kind': 'tmp',
 384                          "capacity": 1073741824 },
 385                 '/var/spool/cwl': {'kind': 'tmp',
 386                                    "capacity": 1073741824 },
 387                 '/var/spool/cwl/foo': {
 388                     'kind': 'collection',
 389                     'path': 'foo',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/foo2': {
 393                     'kind': 'collection',
 394                     'path': 'foo2',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/filename': {
 398                     'kind': 'collection',
 399                     'path': 'filename',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 },
 402                 '/var/spool/cwl/subdir': {
 403                     'kind': 'collection',
 404                     'path': 'subdir',
 405                     'portable_data_hash': '99999999999999999999999999999996+99'
 406                 }
 407             },
 408             'state': 'Committed',
 409             'output_name': 'Output from step test_initial_work_dir',
 410             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 411             'output_path': '/var/spool/cwl',
 412             'output_ttl': 0,
 413             'container_image': '99999999999999999999999999999993+99',
 414             'command': ['ls'],
 415             'cwd': '/var/spool/cwl',
 416             'scheduling_parameters': {
 417             },
 418             'properties': {'cwl_input': {}},
 419             'secret_mounts': {},
 420             'output_storage_classes': ["default"]
 421         }
 422
 423         call_body = call_kwargs.get('body', None)
 424         self.assertNotEqual(None, call_body)
 425         for key in call_body:
 426             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 427
 428
 429     # Test redirecting stdin/stdout/stderr
 430     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 431     def test_redirects(self, keepdocker):
 432         runner = mock.MagicMock()
 433         runner.ignore_docker_for_reuse = False
 434         runner.intermediate_output_ttl = 0
 435         runner.secret_store = cwltool.secrets.SecretStore()
 436         runner.api._rootDesc = {"revision": "20210628"}
 437         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 438
 439         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 440         runner.api.collections().get().execute.return_value = {
 441             "portable_data_hash": "99999999999999999999999999999993+99"}
 442
 443         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 444
 445         tool = cmap({
 446             "inputs": [],
 447             "outputs": [],
 448             "baseCommand": "ls",
 449             "stdout": "stdout.txt",
 450             "stderr": "stderr.txt",
 451             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 452             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 453             "id": "",
 454             "class": "CommandLineTool",
 455             "cwlVersion": "v1.2"
 456         })
 457
 458         loadingContext, runtimeContext = self.helper(runner)
 459         runtimeContext.name = "test_run_redirect"
 460
 461         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 462         arvtool.formatgraph = None
 463         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 464             j.run(runtimeContext)
 465             runner.api.container_requests().create.assert_called_with(
 466                 body=JsonDiffMatcher({
 467                     'environment': {
 468                         'HOME': '/var/spool/cwl',
 469                         'TMPDIR': '/tmp'
 470                     },
 471                     'name': 'test_run_redirect',
 472                     'runtime_constraints': {
 473                         'vcpus': 1,
 474                         'ram': 268435456
 475                     },
 476                     'use_existing': True,
 477                     'priority': 500,
 478                     'mounts': {
 479                         '/tmp': {'kind': 'tmp',
 480                                  "capacity": 1073741824 },
 481                         '/var/spool/cwl': {'kind': 'tmp',
 482                                            "capacity": 1073741824 },
 483                         "stderr": {
 484                             "kind": "file",
 485                             "path": "/var/spool/cwl/stderr.txt"
 486                         },
 487                         "stdin": {
 488                             "kind": "collection",
 489                             "path": "file.txt",
 490                             "portable_data_hash": "99999999999999999999999999999996+99"
 491                         },
 492                         "stdout": {
 493                             "kind": "file",
 494                             "path": "/var/spool/cwl/stdout.txt"
 495                         },
 496                     },
 497                     'state': 'Committed',
 498                     "output_name": "Output from step test_run_redirect",
 499                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 500                     'output_path': '/var/spool/cwl',
 501                     'output_ttl': 0,
 502                     'container_image': '99999999999999999999999999999993+99',
 503                     'command': ['ls', '/var/spool/cwl'],
 504                     'cwd': '/var/spool/cwl',
 505                     'scheduling_parameters': {},
 506                     'properties': {'cwl_input': {}},
 507                     'secret_mounts': {},
 508                     'output_storage_classes': ["default"]
 509                 }))
 510
 511     @mock.patch("arvados.collection.Collection")
 512     def test_done(self, col):
 513         api = mock.MagicMock()
 514
 515         runner = mock.MagicMock()
 516         runner.api = api
 517         runner.num_retries = 0
 518         runner.ignore_docker_for_reuse = False
 519         runner.intermediate_output_ttl = 0
 520         runner.secret_store = cwltool.secrets.SecretStore()
 521
 522         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 523
 524         runner.api.containers().get().execute.return_value = {"state":"Complete",
 525                                                               "output": "abc+123",
 526                                                               "exit_code": 0}
 527
 528         # Need to noop-out the close method otherwise it gets
 529         # discarded when closed and we can't call getvalue() to check
 530         # it.
 531         class NoopCloseStringIO(io.StringIO):
 532             def close(self):
 533                 pass
 534
 535         usage_report = NoopCloseStringIO()
 536         def colreader_action(name, mode):
 537             nonlocal usage_report
 538             if name == "node.json":
 539                 return io.StringIO("""{
 540     "ProviderType": "c5.large",
 541     "VCPUs": 2,
 542     "RAM": 4294967296,
 543     "IncludedScratch": 8000000000000,
 544     "AddedScratch": 0,
 545     "Price": 0.085,
 546     "Preemptible": false,
 547     "CUDA": {
 548         "DriverVersion": "",
 549         "HardwareCapability": "",
 550         "DeviceCount": 0
 551     }
 552 }""")
 553             if name == 'crunchstat.txt':
 554                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 555             if name == 'arv-mount.txt':
 556                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 557             if name == 'usage_report.html':
 558                 return usage_report
 559             return None
 560
 561         col().open.side_effect = colreader_action
 562         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 563
 564         loadingContext, runtimeContext = self.helper(runner)
 565
 566         arvjob = arvados_cwl.ArvadosContainer(runner,
 567                                               runtimeContext,
 568                                               mock.MagicMock(),
 569                                               {},
 570                                               None,
 571                                               [],
 572                                               [],
 573                                               "testjob")
 574         arvjob.output_callback = mock.MagicMock()
 575         arvjob.collect_outputs = mock.MagicMock()
 576         arvjob.successCodes = [0]
 577         arvjob.outdir = "/var/spool/cwl"
 578         arvjob.output_ttl = 3600
 579         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 580
 581         arvjob.collect_outputs.return_value = {"out": "stuff"}
 582
 583         arvjob.done({
 584             "state": "Final",
 585             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 586             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 587             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 588             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 589             "modified_at": "2017-05-26T12:01:22Z",
 590             "properties": {},
 591             "name": "testjob"
 592         })
 593
 594         self.assertFalse(api.collections().create.called)
 595         self.assertFalse(runner.runtime_status_error.called)
 596
 597         # Assert that something was written to the usage report
 598         self.assertTrue(len(usage_report.getvalue()) > 0)
 599
 600         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 601         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 602         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 603
 604         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 605                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 606
 607
 608     # Test to make sure we dont call runtime_status_update if we already did
 609     # some where higher up in the call stack
 610     @mock.patch("arvados_cwl.util.get_current_container")
 611     def test_recursive_runtime_status_update(self, gcc_mock):
 612         self.setup_and_test_container_executor_and_logging(gcc_mock)
 613         root_logger = logging.getLogger('')
 614
 615         # get_current_container is invoked when we call runtime_status_update
 616         # so try and log again!
 617         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 618         try:
 619             root_logger.error("First Error")
 620         except RuntimeError:
 621             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 622
 623
 624     # Test to make sure that an exception raised from
 625     # get_current_container doesn't cause the logger to raise an
 626     # exception
 627     @mock.patch("arvados_cwl.util.get_current_container")
 628     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 629         self.setup_and_test_container_executor_and_logging(gcc_mock)
 630         root_logger = logging.getLogger('')
 631
 632         # get_current_container is invoked when we call
 633         # runtime_status_update, it is going to also raise an
 634         # exception.
 635         gcc_mock.side_effect = Exception("Second Error")
 636         try:
 637             root_logger.error("First Error")
 638         except Exception:
 639             self.fail("Exception in logger should not propagate")
 640         self.assertTrue(gcc_mock.called)
 641
 642     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 643     @mock.patch("arvados_cwl.util.get_current_container")
 644     @mock.patch("arvados.collection.CollectionReader")
 645     @mock.patch("arvados.collection.Collection")
 646     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 647         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 648
 649         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 650         self.assertTrue(gcc_mock.called)
 651
 652         runner.num_retries = 0
 653         runner.ignore_docker_for_reuse = False
 654         runner.intermediate_output_ttl = 0
 655         runner.secret_store = cwltool.secrets.SecretStore()
 656         runner.label = mock.MagicMock()
 657         runner.label.return_value = '[container testjob]'
 658
 659         runner.api.containers().get().execute.return_value = {
 660             "state":"Complete",
 661             "output": "abc+123",
 662             "exit_code": 1,
 663             "log": "def+234"
 664         }
 665
 666         col().open.return_value = []
 667
 668         loadingContext, runtimeContext = self.helper(runner)
 669
 670         arvjob = arvados_cwl.ArvadosContainer(runner,
 671                                               runtimeContext,
 672                                               mock.MagicMock(),
 673                                               {},
 674                                               None,
 675                                               [],
 676                                               [],
 677                                               "testjob")
 678         arvjob.output_callback = mock.MagicMock()
 679         arvjob.collect_outputs = mock.MagicMock()
 680         arvjob.successCodes = [0]
 681         arvjob.outdir = "/var/spool/cwl"
 682         arvjob.output_ttl = 3600
 683         arvjob.collect_outputs.return_value = {"out": "stuff"}
 684
 685         arvjob.done({
 686             "state": "Final",
 687             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 688             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 689             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 690             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 691             "modified_at": "2017-05-26T12:01:22Z",
 692             "properties": {}
 693         })
 694
 695         rts_mock.assert_has_calls([
 696             mock.call('error',
 697                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 698                       '  ** log is empty **'
 699                       ),
 700             mock.call('warning',
 701                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 702         )])
 703         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 704
 705     # The test passes no builder.resources
 706     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 707     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 708     def test_mounts(self, keepdocker):
 709         runner = mock.MagicMock()
 710         runner.ignore_docker_for_reuse = False
 711         runner.intermediate_output_ttl = 0
 712         runner.secret_store = cwltool.secrets.SecretStore()
 713         runner.api._rootDesc = {"revision": "20210628"}
 714         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 715
 716         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 717         runner.api.collections().get().execute.return_value = {
 718             "portable_data_hash": "99999999999999999999999999999994+99",
 719             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 720
 721         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 722
 723         tool = cmap({
 724             "inputs": [
 725                 {"id": "p1",
 726                  "type": "Directory"}
 727             ],
 728             "outputs": [],
 729             "baseCommand": "ls",
 730             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 731             "id": "",
 732             "class": "CommandLineTool",
 733             "cwlVersion": "v1.2"
 734         })
 735
 736         loadingContext, runtimeContext = self.helper(runner)
 737         runtimeContext.name = "test_run_mounts"
 738
 739         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 740         arvtool.formatgraph = None
 741         job_order = {
 742             "p1": {
 743                 "class": "Directory",
 744                 "location": "keep:99999999999999999999999999999994+44",
 745                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 746                 "listing": [
 747                     {
 748                         "class": "File",
 749                         "location": "keep:99999999999999999999999999999994+44/file1",
 750                     },
 751                     {
 752                         "class": "File",
 753                         "location": "keep:99999999999999999999999999999994+44/file2",
 754                     }
 755                 ]
 756             }
 757         }
 758         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 759             j.run(runtimeContext)
 760             runner.api.container_requests().create.assert_called_with(
 761                 body=JsonDiffMatcher({
 762                     'environment': {
 763                         'HOME': '/var/spool/cwl',
 764                         'TMPDIR': '/tmp'
 765                     },
 766                     'name': 'test_run_mounts',
 767                     'runtime_constraints': {
 768                         'vcpus': 1,
 769                         'ram': 268435456
 770                     },
 771                     'use_existing': True,
 772                     'priority': 500,
 773                     'mounts': {
 774                         "/keep/99999999999999999999999999999994+44": {
 775                             "kind": "collection",
 776                             "portable_data_hash": "99999999999999999999999999999994+44",
 777                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 778                         },
 779                         '/tmp': {'kind': 'tmp',
 780                                  "capacity": 1073741824 },
 781                         '/var/spool/cwl': {'kind': 'tmp',
 782                                            "capacity": 1073741824 }
 783                     },
 784                     'state': 'Committed',
 785                     'output_name': 'Output from step test_run_mounts',
 786                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 787                     'output_path': '/var/spool/cwl',
 788                     'output_ttl': 0,
 789                     'container_image': '99999999999999999999999999999994+99',
 790                     'command': ['ls', '/var/spool/cwl'],
 791                     'cwd': '/var/spool/cwl',
 792                     'scheduling_parameters': {},
 793                     'properties': {'cwl_input': {
 794                         "p1": {
 795                             "basename": "99999999999999999999999999999994+44",
 796                             "class": "Directory",
 797                             "dirname": "/keep",
 798                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 799                             "listing": [
 800                                 {
 801                                     "basename": "file1",
 802                                     "class": "File",
 803                                     "dirname": "/keep/99999999999999999999999999999994+44",
 804                                     "location": "keep:99999999999999999999999999999994+44/file1",
 805                                     "nameext": "",
 806                                     "nameroot": "file1",
 807                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 808                                     "size": 0
 809                                 },
 810                                 {
 811                                     "basename": "file2",
 812                                     "class": "File",
 813                                     "dirname": "/keep/99999999999999999999999999999994+44",
 814                                     "location": "keep:99999999999999999999999999999994+44/file2",
 815                                     "nameext": "",
 816                                     "nameroot": "file2",
 817                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 818                                     "size": 0
 819                                 }
 820                             ],
 821                             "location": "keep:99999999999999999999999999999994+44",
 822                             "path": "/keep/99999999999999999999999999999994+44"
 823                         }
 824                     }},
 825                     'secret_mounts': {},
 826                     'output_storage_classes': ["default"]
 827                 }))
 828
 829     # The test passes no builder.resources
 830     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 831     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 832     def test_secrets(self, keepdocker):
 833         arvados_cwl.add_arv_hints()
 834         runner = mock.MagicMock()
 835         runner.ignore_docker_for_reuse = False
 836         runner.intermediate_output_ttl = 0
 837         runner.secret_store = cwltool.secrets.SecretStore()
 838         runner.api._rootDesc = {"revision": "20210628"}
 839         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 840
 841         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 842         runner.api.collections().get().execute.return_value = {
 843             "portable_data_hash": "99999999999999999999999999999993+99"}
 844
 845         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 846
 847         tool = cmap({"arguments": ["md5sum", "example.conf"],
 848                      "class": "CommandLineTool",
 849                      "cwlVersion": "v1.2",
 850                      "hints": [
 851                          {
 852                              "class": "http://commonwl.org/cwltool#Secrets",
 853                              "secrets": [
 854                                  "#secret_job.cwl/pw"
 855                              ]
 856                          }
 857                      ],
 858                      "id": "",
 859                      "inputs": [
 860                          {
 861                              "id": "#secret_job.cwl/pw",
 862                              "type": "string"
 863                          }
 864                      ],
 865                      "outputs": [
 866                      ],
 867                      "requirements": [
 868                          {
 869                              "class": "InitialWorkDirRequirement",
 870                              "listing": [
 871                                  {
 872                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 873                                      "entryname": "example.conf"
 874                                  }
 875                              ]
 876                          }
 877                      ]})
 878
 879         loadingContext, runtimeContext = self.helper(runner)
 880         runtimeContext.name = "test_secrets"
 881
 882         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 883         arvtool.formatgraph = None
 884
 885         job_order = {"pw": "blorp"}
 886         runner.secret_store.store(["pw"], job_order)
 887
 888         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 889             j.run(runtimeContext)
 890             runner.api.container_requests().create.assert_called_with(
 891                 body=JsonDiffMatcher({
 892                     'environment': {
 893                         'HOME': '/var/spool/cwl',
 894                         'TMPDIR': '/tmp'
 895                     },
 896                     'name': 'test_secrets',
 897                     'runtime_constraints': {
 898                         'vcpus': 1,
 899                         'ram': 268435456
 900                     },
 901                     'use_existing': True,
 902                     'priority': 500,
 903                     'mounts': {
 904                         '/tmp': {'kind': 'tmp',
 905                                  "capacity": 1073741824
 906                              },
 907                         '/var/spool/cwl': {'kind': 'tmp',
 908                                            "capacity": 1073741824 }
 909                     },
 910                     'state': 'Committed',
 911                     'output_name': 'Output from step test_secrets',
 912                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 913                     'output_path': '/var/spool/cwl',
 914                     'output_ttl': 0,
 915                     'container_image': '99999999999999999999999999999993+99',
 916                     'command': ['md5sum', 'example.conf'],
 917                     'cwd': '/var/spool/cwl',
 918                     'scheduling_parameters': {},
 919                     'properties': {'cwl_input': job_order},
 920                     "secret_mounts": {
 921                         "/var/spool/cwl/example.conf": {
 922                             "content": "username: user\npassword: blorp\n",
 923                             "kind": "text"
 924                         }
 925                     },
 926                     'output_storage_classes': ["default"]
 927                 }))
 928
 929     # The test passes no builder.resources
 930     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 931     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 932     def test_timelimit(self, keepdocker):
 933         runner = mock.MagicMock()
 934         runner.ignore_docker_for_reuse = False
 935         runner.intermediate_output_ttl = 0
 936         runner.secret_store = cwltool.secrets.SecretStore()
 937         runner.api._rootDesc = {"revision": "20210628"}
 938         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 939
 940         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 941         runner.api.collections().get().execute.return_value = {
 942             "portable_data_hash": "99999999999999999999999999999993+99"}
 943
 944         tool = cmap({
 945             "inputs": [],
 946             "outputs": [],
 947             "baseCommand": "ls",
 948             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 949             "id": "",
 950             "cwlVersion": "v1.2",
 951             "class": "CommandLineTool",
 952             "hints": [
 953                 {
 954                     "class": "ToolTimeLimit",
 955                     "timelimit": 42
 956                 }
 957             ]
 958         })
 959
 960         loadingContext, runtimeContext = self.helper(runner)
 961         runtimeContext.name = "test_timelimit"
 962
 963         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 964         arvtool.formatgraph = None
 965
 966         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 967             j.run(runtimeContext)
 968
 969         _, kwargs = runner.api.container_requests().create.call_args
 970         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 971
 972
 973     # The test passes no builder.resources
 974     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 975     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 976     def test_setting_storage_class(self, keepdocker):
 977         #arv_docker_clear_cache()
 978
 979         runner = mock.MagicMock()
 980         runner.ignore_docker_for_reuse = False
 981         runner.intermediate_output_ttl = 0
 982         runner.secret_store = cwltool.secrets.SecretStore()
 983         runner.api._rootDesc = {"revision": "20210628"}
 984         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 985
 986         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 987         runner.api.collections().get().execute.return_value = {
 988             "portable_data_hash": "99999999999999999999999999999993+99"}
 989
 990         tool = cmap({
 991             "inputs": [],
 992             "outputs": [],
 993             "baseCommand": "ls",
 994             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 995             "id": "",
 996             "cwlVersion": "v1.2",
 997             "class": "CommandLineTool",
 998             "hints": [
 999                 {
1000                     "class": "http://arvados.org/cwl#OutputStorageClass",
1001                     "finalStorageClass": ["baz_sc", "qux_sc"],
1002                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1003                 }
1004             ]
1005         })
1006
1007         loadingContext, runtimeContext = self.helper(runner, True)
1008
1009         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1010         arvtool.formatgraph = None
1011
1012         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1013             j.run(runtimeContext)
1014             runner.api.container_requests().create.assert_called_with(
1015                 body=JsonDiffMatcher({
1016                     'environment': {
1017                         'HOME': '/var/spool/cwl',
1018                         'TMPDIR': '/tmp'
1019                     },
1020                     'name': 'test_run_True',
1021                     'runtime_constraints': {
1022                         'vcpus': 1,
1023                         'ram': 268435456
1024                     },
1025                     'use_existing': True,
1026                     'priority': 500,
1027                     'mounts': {
1028                         '/tmp': {'kind': 'tmp',
1029                                  "capacity": 1073741824
1030                              },
1031                         '/var/spool/cwl': {'kind': 'tmp',
1032                                            "capacity": 1073741824 }
1033                     },
1034                     'state': 'Committed',
1035                     'output_name': 'Output from step test_run_True',
1036                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1037                     'output_path': '/var/spool/cwl',
1038                     'output_ttl': 0,
1039                     'container_image': '99999999999999999999999999999993+99',
1040                     'command': ['ls', '/var/spool/cwl'],
1041                     'cwd': '/var/spool/cwl',
1042                     'scheduling_parameters': {},
1043                     'properties': {'cwl_input': {}},
1044                     'secret_mounts': {},
1045                     'output_storage_classes': ["foo_sc", "bar_sc"]
1046                 }))
1047
1048
1049     # The test passes no builder.resources
1050     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1051     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1052     def test_setting_process_properties(self, keepdocker):
1053         #arv_docker_clear_cache()
1054
1055         runner = mock.MagicMock()
1056         runner.ignore_docker_for_reuse = False
1057         runner.intermediate_output_ttl = 0
1058         runner.secret_store = cwltool.secrets.SecretStore()
1059         runner.api._rootDesc = {"revision": "20210628"}
1060         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1061
1062         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1063         runner.api.collections().get().execute.return_value = {
1064             "portable_data_hash": "99999999999999999999999999999993+99"}
1065
1066         tool = cmap({
1067             "inputs": [
1068                 {"id": "x", "type": "string"}],
1069             "outputs": [],
1070             "baseCommand": "ls",
1071             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1072             "id": "",
1073             "class": "CommandLineTool",
1074             "cwlVersion": "v1.2",
1075             "hints": [
1076             {
1077                 "class": "http://arvados.org/cwl#ProcessProperties",
1078                 "processProperties": [
1079                     {"propertyName": "foo",
1080                      "propertyValue": "bar"},
1081                     {"propertyName": "baz",
1082                      "propertyValue": "$(inputs.x)"},
1083                     {"propertyName": "quux",
1084                      "propertyValue": {
1085                          "q1": 1,
1086                          "q2": 2
1087                      }
1088                     }
1089                 ],
1090             }
1091         ]
1092         })
1093
1094         loadingContext, runtimeContext = self.helper(runner, True)
1095
1096         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1097         arvtool.formatgraph = None
1098
1099         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1100             j.run(runtimeContext)
1101             runner.api.container_requests().create.assert_called_with(
1102                 body=JsonDiffMatcher({
1103                     'environment': {
1104                         'HOME': '/var/spool/cwl',
1105                         'TMPDIR': '/tmp'
1106                     },
1107                     'name': 'test_run_True',
1108                     'runtime_constraints': {
1109                         'vcpus': 1,
1110                         'ram': 268435456
1111                     },
1112                     'use_existing': True,
1113                     'priority': 500,
1114                     'mounts': {
1115                         '/tmp': {'kind': 'tmp',
1116                                  "capacity": 1073741824
1117                              },
1118                         '/var/spool/cwl': {'kind': 'tmp',
1119                                            "capacity": 1073741824 }
1120                     },
1121                     'state': 'Committed',
1122                     'output_name': 'Output from step test_run_True',
1123                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1124                     'output_path': '/var/spool/cwl',
1125                     'output_ttl': 0,
1126                     'container_image': '99999999999999999999999999999993+99',
1127                     'command': ['ls', '/var/spool/cwl'],
1128                     'cwd': '/var/spool/cwl',
1129                     'scheduling_parameters': {},
1130                     'properties': {
1131                         "baz": "blorp",
1132                         "cwl_input": {"x": "blorp"},
1133                         "foo": "bar",
1134                         "quux": {
1135                             "q1": 1,
1136                             "q2": 2
1137                         }
1138                     },
1139                     'secret_mounts': {},
1140                     'output_storage_classes': ["default"]
1141                 }))
1142
1143
1144     # The test passes no builder.resources
1145     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1146     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1147     def test_cuda_requirement(self, keepdocker):
1148         arvados_cwl.add_arv_hints()
1149         #arv_docker_clear_cache()
1150
1151         runner = mock.MagicMock()
1152         runner.ignore_docker_for_reuse = False
1153         runner.intermediate_output_ttl = 0
1154         runner.secret_store = cwltool.secrets.SecretStore()
1155         runner.api._rootDesc = {"revision": "20210628"}
1156         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1157
1158         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1159         runner.api.collections().get().execute.return_value = {
1160             "portable_data_hash": "99999999999999999999999999999993+99"}
1161
1162         test_cwl_req = [{
1163                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1164                 "cudaVersionMin": "11.0",
1165                 "cudaComputeCapability": "9.0",
1166             }, {
1167                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1168                 "cudaVersionMin": "11.0",
1169                 "cudaComputeCapability": "9.0",
1170                 "cudaDeviceCountMin": 2
1171             }, {
1172                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1173                 "cudaVersionMin": "11.0",
1174                 "cudaComputeCapability": ["4.0", "5.0"],
1175                 "cudaDeviceCountMin": 2
1176             }]
1177
1178         test_arv_req = [{
1179             'device_count': 1,
1180             'driver_version': "11.0",
1181             'hardware_capability': "9.0"
1182         }, {
1183             'device_count': 2,
1184             'driver_version': "11.0",
1185             'hardware_capability': "9.0"
1186         }, {
1187             'device_count': 2,
1188             'driver_version': "11.0",
1189             'hardware_capability': "4.0"
1190         }]
1191
1192         for test_case in range(0, len(test_cwl_req)):
1193
1194             tool = cmap({
1195                 "inputs": [],
1196                 "outputs": [],
1197                 "baseCommand": "nvidia-smi",
1198                 "arguments": [],
1199                 "id": "",
1200                 "cwlVersion": "v1.2",
1201                 "class": "CommandLineTool",
1202                 "requirements": [test_cwl_req[test_case]]
1203             })
1204
1205             loadingContext, runtimeContext = self.helper(runner, True)
1206
1207             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1208             arvtool.formatgraph = None
1209
1210             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1211                 j.run(runtimeContext)
1212                 runner.api.container_requests().create.assert_called_with(
1213                     body=JsonDiffMatcher({
1214                         'environment': {
1215                             'HOME': '/var/spool/cwl',
1216                             'TMPDIR': '/tmp'
1217                         },
1218                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1219                         'runtime_constraints': {
1220                             'vcpus': 1,
1221                             'ram': 268435456,
1222                             'cuda': test_arv_req[test_case]
1223                         },
1224                         'use_existing': True,
1225                         'priority': 500,
1226                         'mounts': {
1227                             '/tmp': {'kind': 'tmp',
1228                                      "capacity": 1073741824
1229                                  },
1230                             '/var/spool/cwl': {'kind': 'tmp',
1231                                                "capacity": 1073741824 }
1232                         },
1233                         'state': 'Committed',
1234                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1235                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1236                         'output_path': '/var/spool/cwl',
1237                         'output_ttl': 0,
1238                         'container_image': '99999999999999999999999999999993+99',
1239                         'command': ['nvidia-smi'],
1240                         'cwd': '/var/spool/cwl',
1241                         'scheduling_parameters': {},
1242                         'properties': {'cwl_input': {}},
1243                         'secret_mounts': {},
1244                         'output_storage_classes': ["default"]
1245                     }))
1246
1247
1248     # The test passes no builder.resources
1249     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1250     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1251     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1252     def test_match_local_docker(self, keepdocker, determine_image_id):
1253         arvados_cwl.add_arv_hints()
1254
1255         runner = mock.MagicMock()
1256         runner.ignore_docker_for_reuse = False
1257         runner.intermediate_output_ttl = 0
1258         runner.secret_store = cwltool.secrets.SecretStore()
1259         runner.api._rootDesc = {"revision": "20210628"}
1260         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1261
1262         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1263                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1264         determine_image_id.side_effect = lambda x: "123"
1265         def execute(uuid):
1266             ex = mock.MagicMock()
1267             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1268                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1269             ex.execute.return_value = lookup[uuid]
1270             return ex
1271         runner.api.collections().get.side_effect = execute
1272
1273         tool = cmap({
1274             "inputs": [],
1275             "outputs": [],
1276             "baseCommand": "echo",
1277             "arguments": [],
1278             "id": "",
1279             "cwlVersion": "v1.0",
1280             "class": "org.w3id.cwl.cwl.CommandLineTool"
1281         })
1282
1283         loadingContext, runtimeContext = self.helper(runner, True)
1284
1285         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1286         arvtool.formatgraph = None
1287
1288         container_request = {
1289             'environment': {
1290                 'HOME': '/var/spool/cwl',
1291                 'TMPDIR': '/tmp'
1292             },
1293             'name': 'test_run_True',
1294             'runtime_constraints': {
1295                 'vcpus': 1,
1296                 'ram': 1073741824,
1297             },
1298             'use_existing': True,
1299             'priority': 500,
1300             'mounts': {
1301                 '/tmp': {'kind': 'tmp',
1302                          "capacity": 1073741824
1303                          },
1304                 '/var/spool/cwl': {'kind': 'tmp',
1305                                    "capacity": 1073741824 }
1306             },
1307             'state': 'Committed',
1308             'output_name': 'Output from step test_run_True',
1309             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1310             'output_path': '/var/spool/cwl',
1311             'output_ttl': 0,
1312             'container_image': '99999999999999999999999999999994+99',
1313             'command': ['echo'],
1314             'cwd': '/var/spool/cwl',
1315             'scheduling_parameters': {},
1316             'properties': {'cwl_input': {}},
1317             'secret_mounts': {},
1318             'output_storage_classes': ["default"]
1319         }
1320
1321         runtimeContext.match_local_docker = False
1322         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1323             j.run(runtimeContext)
1324             runner.api.container_requests().create.assert_called_with(
1325                 body=JsonDiffMatcher(container_request))
1326
1327         runtimeContext.cached_docker_lookups.clear()
1328         runtimeContext.match_local_docker = True
1329         container_request['container_image'] = '99999999999999999999999999999993+99'
1330         container_request['name'] = 'test_run_True_2'
1331         container_request['output_name'] = 'Output from step test_run_True_2'
1332         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1333             j.run(runtimeContext)
1334             runner.api.container_requests().create.assert_called_with(
1335                 body=JsonDiffMatcher(container_request))
1336
1337
1338     # The test passes no builder.resources
1339     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1340     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1341     def test_run_preemptible_hint(self, keepdocker):
1342         arvados_cwl.add_arv_hints()
1343         for enable_preemptible in (None, True, False):
1344             for preemptible_hint in (None, True, False):
1345                 #arv_docker_clear_cache()
1346
1347                 runner = mock.MagicMock()
1348                 runner.ignore_docker_for_reuse = False
1349                 runner.intermediate_output_ttl = 0
1350                 runner.secret_store = cwltool.secrets.SecretStore()
1351                 runner.api._rootDesc = {"revision": "20210628"}
1352                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1353
1354                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1355                 runner.api.collections().get().execute.return_value = {
1356                     "portable_data_hash": "99999999999999999999999999999993+99"}
1357
1358                 if preemptible_hint is not None:
1359                     hints = [{
1360                         "class": "http://arvados.org/cwl#UsePreemptible",
1361                         "usePreemptible": preemptible_hint
1362                     }]
1363                 else:
1364                     hints = []
1365
1366                 tool = cmap({
1367                     "inputs": [],
1368                     "outputs": [],
1369                     "baseCommand": "ls",
1370                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1371                     "id": "",
1372                     "class": "CommandLineTool",
1373                     "cwlVersion": "v1.2",
1374                     "hints": hints
1375                 })
1376
1377                 loadingContext, runtimeContext = self.helper(runner)
1378
1379                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1380                 runtimeContext.enable_preemptible = enable_preemptible
1381
1382                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1383                 arvtool.formatgraph = None
1384
1385                 # Test the interactions between --enable/disable-preemptible
1386                 # and UsePreemptible hint
1387
1388                 if enable_preemptible is None:
1389                     if preemptible_hint is None:
1390                         sched = {}
1391                     else:
1392                         sched = {'preemptible': preemptible_hint}
1393                 else:
1394                     if preemptible_hint is None:
1395                         sched = {'preemptible': enable_preemptible}
1396                     else:
1397                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1398
1399                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1400                     j.run(runtimeContext)
1401                     runner.api.container_requests().create.assert_called_with(
1402                         body=JsonDiffMatcher({
1403                             'environment': {
1404                                 'HOME': '/var/spool/cwl',
1405                                 'TMPDIR': '/tmp'
1406                             },
1407                             'name': runtimeContext.name,
1408                             'runtime_constraints': {
1409                                 'vcpus': 1,
1410                                 'ram': 268435456
1411                             },
1412                             'use_existing': True,
1413                             'priority': 500,
1414                             'mounts': {
1415                                 '/tmp': {'kind': 'tmp',
1416                                          "capacity": 1073741824
1417                                      },
1418                                 '/var/spool/cwl': {'kind': 'tmp',
1419                                                    "capacity": 1073741824 }
1420                             },
1421                             'state': 'Committed',
1422                             'output_name': 'Output from step '+runtimeContext.name,
1423                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1424                             'output_path': '/var/spool/cwl',
1425                             'output_ttl': 0,
1426                             'container_image': '99999999999999999999999999999993+99',
1427                             'command': ['ls', '/var/spool/cwl'],
1428                             'cwd': '/var/spool/cwl',
1429                             'scheduling_parameters': sched,
1430                             'properties': {'cwl_input': {}},
1431                             'secret_mounts': {},
1432                             'output_storage_classes': ["default"]
1433                         }))
1434
1435
1436     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1437     def test_output_properties(self, keepdocker):
1438         arvados_cwl.add_arv_hints()
1439         for rev in ["20210628", "20220510"]:
1440             runner = mock.MagicMock()
1441             runner.ignore_docker_for_reuse = False
1442             runner.intermediate_output_ttl = 0
1443             runner.secret_store = cwltool.secrets.SecretStore()
1444             runner.api._rootDesc = {"revision": rev}
1445             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1446
1447             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1448             runner.api.collections().get().execute.return_value = {
1449                 "portable_data_hash": "99999999999999999999999999999993+99"}
1450
1451             tool = cmap({
1452                 "inputs": [{
1453                     "id": "inp",
1454                     "type": "string"
1455                 }],
1456                 "outputs": [],
1457                 "baseCommand": "ls",
1458                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1459                 "id": "",
1460                 "cwlVersion": "v1.2",
1461                 "class": "CommandLineTool",
1462                 "hints": [
1463                     {
1464                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1465                         "outputProperties": {
1466                             "foo": "bar",
1467                             "baz": "$(inputs.inp)"
1468                         }
1469                     }
1470                 ]
1471             })
1472
1473             loadingContext, runtimeContext = self.helper(runner)
1474             runtimeContext.name = "test_timelimit"
1475
1476             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1477             arvtool.formatgraph = None
1478
1479             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1480                 j.run(runtimeContext)
1481
1482             _, kwargs = runner.api.container_requests().create.call_args
1483             if rev == "20220510":
1484                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1485             else:
1486                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1487
1488
1489 class TestWorkflow(unittest.TestCase):
1490     def setUp(self):
1491         cwltool.process._names = set()
1492         #arv_docker_clear_cache()
1493
1494     def helper(self, runner, enable_reuse=True):
1495         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1496
1497         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1498                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1499
1500         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1501         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1502         document_loader.fetch_text = document_loader.fetcher.fetch_text
1503         document_loader.check_exists = document_loader.fetcher.check_exists
1504
1505         loadingContext = arvados_cwl.context.ArvLoadingContext(
1506             {"avsc_names": avsc_names,
1507              "basedir": "",
1508              "make_fs_access": make_fs_access,
1509              "loader": document_loader,
1510              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1511              "construct_tool_object": runner.arv_make_tool,
1512              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
1513         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1514             {"work_api": "containers",
1515              "basedir": "",
1516              "name": "test_run_wf_"+str(enable_reuse),
1517              "make_fs_access": make_fs_access,
1518              "tmpdir": "/tmp",
1519              "enable_reuse": enable_reuse,
1520              "priority": 500})
1521
1522         return loadingContext, runtimeContext
1523
1524     # The test passes no builder.resources
1525     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1526     @mock.patch("arvados.collection.CollectionReader")
1527     @mock.patch("arvados.collection.Collection")
1528     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1529     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1530         arvados_cwl.add_arv_hints()
1531
1532         api = mock.MagicMock()
1533         api._rootDesc = get_rootDesc()
1534         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1535
1536         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1537         self.assertEqual(runner.work_api, 'containers')
1538
1539         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1540         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1541         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1542                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1543
1544         runner.api.containers().current().execute.return_value = {}
1545
1546         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1547         runner.ignore_docker_for_reuse = False
1548         runner.num_retries = 0
1549         runner.secret_store = cwltool.secrets.SecretStore()
1550
1551         loadingContext, runtimeContext = self.helper(runner)
1552         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1553
1554         mockcollectionreader().exists.return_value = True
1555
1556         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1557         metadata["cwlVersion"] = tool["cwlVersion"]
1558
1559         mockc = mock.MagicMock()
1560         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1561         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1562
1563         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1564         arvtool.formatgraph = None
1565         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1566
1567         next(it).run(runtimeContext)
1568         next(it).run(runtimeContext)
1569
1570         with open("tests/wf/scatter2_subwf.cwl") as f:
1571             subwf = StripYAMLComments(f.read()).rstrip()
1572
1573         runner.api.container_requests().create.assert_called_with(
1574             body=JsonDiffMatcher({
1575                 "command": [
1576                     "cwltool",
1577                     "--no-container",
1578                     "--move-outputs",
1579                     "--preserve-entire-environment",
1580                     "workflow.cwl",
1581                     "cwl.input.yml"
1582                 ],
1583                 "container_image": "99999999999999999999999999999993+99",
1584                 "cwd": "/var/spool/cwl",
1585                 "environment": {
1586                     "HOME": "/var/spool/cwl",
1587                     "TMPDIR": "/tmp"
1588                 },
1589                 "mounts": {
1590                     "/keep/99999999999999999999999999999999+118": {
1591                         "kind": "collection",
1592                         "portable_data_hash": "99999999999999999999999999999999+118"
1593                     },
1594                     "/tmp": {
1595                         "capacity": 1073741824,
1596                         "kind": "tmp"
1597                     },
1598                     "/var/spool/cwl": {
1599                         "capacity": 1073741824,
1600                         "kind": "tmp"
1601                     },
1602                     "/var/spool/cwl/cwl.input.yml": {
1603                         "kind": "collection",
1604                         "path": "cwl.input.yml",
1605                         "portable_data_hash": "99999999999999999999999999999996+99"
1606                     },
1607                     "/var/spool/cwl/workflow.cwl": {
1608                         "kind": "collection",
1609                         "path": "workflow.cwl",
1610                         "portable_data_hash": "99999999999999999999999999999996+99"
1611                     },
1612                     "stdout": {
1613                         "kind": "file",
1614                         "path": "/var/spool/cwl/cwl.output.json"
1615                     }
1616                 },
1617                 "name": "scatterstep",
1618                 "output_name": "Output from step scatterstep",
1619                 "output_path": "/var/spool/cwl",
1620                 "output_ttl": 0,
1621                 "priority": 500,
1622                 "properties": {'cwl_input': {
1623                         "fileblub": {
1624                             "basename": "token.txt",
1625                             "class": "File",
1626                             "dirname": "/keep/99999999999999999999999999999999+118",
1627                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1628                             "nameext": ".txt",
1629                             "nameroot": "token",
1630                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1631                             "size": 0
1632                         },
1633                         "sleeptime": 5
1634                 }},
1635                 "runtime_constraints": {
1636                     "ram": 1073741824,
1637                     "vcpus": 1
1638                 },
1639                 "scheduling_parameters": {},
1640                 "secret_mounts": {},
1641                 "state": "Committed",
1642                 "use_existing": True,
1643                 'output_storage_classes': ["default"]
1644             }))
1645         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1646         mockc.open().__enter__().write.assert_has_calls([mock.call(
1647 '''{
1648   "fileblub": {
1649     "basename": "token.txt",
1650     "class": "File",
1651     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1652     "size": 0
1653   },
1654   "sleeptime": 5
1655 }''')])
1656
1657     # The test passes no builder.resources
1658     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1659     @mock.patch("arvados.collection.CollectionReader")
1660     @mock.patch("arvados.collection.Collection")
1661     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1662     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1663         arvados_cwl.add_arv_hints()
1664
1665         api = mock.MagicMock()
1666         api._rootDesc = get_rootDesc()
1667         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1668
1669         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1670         self.assertEqual(runner.work_api, 'containers')
1671
1672         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1673         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1674                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1675         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1676                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1677
1678         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1679         runner.ignore_docker_for_reuse = False
1680         runner.num_retries = 0
1681         runner.secret_store = cwltool.secrets.SecretStore()
1682
1683         loadingContext, runtimeContext = self.helper(runner)
1684         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1685         loadingContext.do_update = True
1686         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1687
1688         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1689
1690         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1691         arvtool.formatgraph = None
1692         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1693
1694         next(it).run(runtimeContext)
1695         next(it).run(runtimeContext)
1696
1697         with open("tests/wf/echo-subwf.cwl") as f:
1698             subwf = StripYAMLComments(f.read())
1699
1700         runner.api.container_requests().create.assert_called_with(
1701             body=JsonDiffMatcher({
1702                 'output_ttl': 0,
1703                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1704                 'scheduling_parameters': {},
1705                 'name': u'echo-subwf',
1706                 'secret_mounts': {},
1707                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1708                 'properties': {'cwl_input': {}},
1709                 'priority': 500,
1710                 'mounts': {
1711                     '/var/spool/cwl/cwl.input.yml': {
1712                         'portable_data_hash': '99999999999999999999999999999996+99',
1713                         'kind': 'collection',
1714                         'path': 'cwl.input.yml'
1715                     },
1716                     '/var/spool/cwl/workflow.cwl': {
1717                         'portable_data_hash': '99999999999999999999999999999996+99',
1718                         'kind': 'collection',
1719                         'path': 'workflow.cwl'
1720                     },
1721                     'stdout': {
1722                         'path': '/var/spool/cwl/cwl.output.json',
1723                         'kind': 'file'
1724                     },
1725                     '/tmp': {
1726                         'kind': 'tmp',
1727                         'capacity': 1073741824
1728                     }, '/var/spool/cwl': {
1729                         'kind': 'tmp',
1730                         'capacity': 3221225472
1731                     }
1732                 },
1733                 'state': 'Committed',
1734                 'output_path': '/var/spool/cwl',
1735                 'container_image': '99999999999999999999999999999993+99',
1736                 'command': [
1737                     u'cwltool',
1738                     u'--no-container',
1739                     u'--move-outputs',
1740                     u'--preserve-entire-environment',
1741                     u'workflow.cwl',
1742                     u'cwl.input.yml'
1743                 ],
1744                 'use_existing': True,
1745                 'output_name': u'Output from step echo-subwf',
1746                 'cwd': '/var/spool/cwl',
1747                 'output_storage_classes': ["default"]
1748             }))
1749
1750     def test_default_work_api(self):
1751         arvados_cwl.add_arv_hints()
1752
1753         api = mock.MagicMock()
1754         api._rootDesc = copy.deepcopy(get_rootDesc())
1755         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1756         self.assertEqual(runner.work_api, 'containers')