sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 #from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import unittest
  16 import os
  17 import functools
  18 import threading
  19 import cwltool.process
  20 import cwltool.secrets
  21 import cwltool.load_tool
  22 from cwltool.update import INTERNAL_VERSION
  23 from schema_salad.ref_resolver import Loader
  24 from schema_salad.sourceline import cmap
  25 import io
  26
  27 from unittest import mock
  28
  29 from .matcher import JsonDiffMatcher, StripYAMLComments
  30 from .mock_discovery import get_rootDesc
  31
  32 if not os.getenv('ARVADOS_DEBUG'):
  33     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  34     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  35
  36 class CollectionMock(object):
  37     def __init__(self, vwdmock, *args, **kwargs):
  38         self.vwdmock = vwdmock
  39         self.count = 0
  40
  41     def open(self, *args, **kwargs):
  42         self.count += 1
  43         return self.vwdmock.open(*args, **kwargs)
  44
  45     def copy(self, *args, **kwargs):
  46         self.count += 1
  47         self.vwdmock.copy(*args, **kwargs)
  48
  49     def save_new(self, *args, **kwargs):
  50         pass
  51
  52     def __len__(self):
  53         return self.count
  54
  55     def portable_data_hash(self):
  56         if self.count == 0:
  57             return arvados.config.EMPTY_BLOCK_LOCATOR
  58         else:
  59             return "99999999999999999999999999999996+99"
  60
  61
  62 class TestContainer(unittest.TestCase):
  63
  64     def setUp(self):
  65         cwltool.process._names = set()
  66         #arv_docker_clear_cache()
  67
  68     def tearDown(self):
  69         root_logger = logging.getLogger('')
  70
  71         # Remove existing RuntimeStatusLoggingHandlers if they exist
  72         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  73         root_logger.handlers = handlers
  74
  75     def helper(self, runner, enable_reuse=True):
  76         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  77
  78         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  79                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  80         fs_access = mock.MagicMock()
  81         fs_access.exists.return_value = True
  82
  83         loadingContext = arvados_cwl.context.ArvLoadingContext(
  84             {"avsc_names": avsc_names,
  85              "basedir": "",
  86              "make_fs_access": make_fs_access,
  87              "construct_tool_object": runner.arv_make_tool,
  88              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  89              "loader": Loader({}),
  90              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  91              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
  92              })
  93         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  94             {"work_api": "containers",
  95              "basedir": "",
  96              "name": "test_run_"+str(enable_reuse),
  97              "make_fs_access": make_fs_access,
  98              "tmpdir": "/tmp",
  99              "outdir": "/tmp",
 100              "enable_reuse": enable_reuse,
 101              "priority": 500,
 102              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 103              "workflow_eval_lock": threading.Condition(threading.RLock())
 104             })
 105
 106         if isinstance(runner, mock.MagicMock):
 107             def make_tool(toolpath_object, loadingContext):
 108                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 109             runner.arv_make_tool.side_effect = make_tool
 110
 111         return loadingContext, runtimeContext
 112
 113     # Helper function to set up the ArvCwlExecutor to use the containers api
 114     # and test that the RuntimeStatusLoggingHandler is set up correctly
 115     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 116         api = mock.MagicMock()
 117         api._rootDesc = copy.deepcopy(get_rootDesc())
 118
 119         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 120         # adds the logging handler that will call runtime_status_update() mock
 121         self.assertFalse(gcc_mock.called)
 122         runner = arvados_cwl.ArvCwlExecutor(api)
 123         self.assertEqual(runner.work_api, 'containers')
 124         root_logger = logging.getLogger('')
 125         handlerClasses = [h.__class__ for h in root_logger.handlers]
 126         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 127         return runner
 128
 129     # The test passes no builder.resources
 130     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 131     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 132     def test_run(self, keepdocker):
 133         for enable_reuse in (True, False):
 134             #arv_docker_clear_cache()
 135
 136             runner = mock.MagicMock()
 137             runner.ignore_docker_for_reuse = False
 138             runner.intermediate_output_ttl = 0
 139             runner.secret_store = cwltool.secrets.SecretStore()
 140             runner.api._rootDesc = {"revision": "20210628"}
 141             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 142
 143             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 144             runner.api.collections().get().execute.return_value = {
 145                 "portable_data_hash": "99999999999999999999999999999993+99"}
 146
 147             tool = cmap({
 148                 "inputs": [],
 149                 "outputs": [],
 150                 "baseCommand": "ls",
 151                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 152                 "id": "",
 153                 "class": "CommandLineTool",
 154                 "cwlVersion": "v1.2"
 155             })
 156
 157             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 158
 159             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 160             arvtool.formatgraph = None
 161
 162             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 163                 j.run(runtimeContext)
 164                 runner.api.container_requests().create.assert_called_with(
 165                     body=JsonDiffMatcher({
 166                         'environment': {
 167                             'HOME': '/var/spool/cwl',
 168                             'TMPDIR': '/tmp'
 169                         },
 170                         'name': 'test_run_'+str(enable_reuse),
 171                         'runtime_constraints': {
 172                             'vcpus': 1,
 173                             'ram': 268435456
 174                         },
 175                         'use_existing': enable_reuse,
 176                         'priority': 500,
 177                         'mounts': {
 178                             '/tmp': {'kind': 'tmp',
 179                                      "capacity": 1073741824
 180                                  },
 181                             '/var/spool/cwl': {'kind': 'tmp',
 182                                                "capacity": 1073741824 }
 183                         },
 184                         'state': 'Committed',
 185                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 186                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 187                         'output_path': '/var/spool/cwl',
 188                         'output_ttl': 0,
 189                         'container_image': '99999999999999999999999999999993+99',
 190                         'command': ['ls', '/var/spool/cwl'],
 191                         'cwd': '/var/spool/cwl',
 192                         'scheduling_parameters': {},
 193                         'properties': {'cwl_input': {}},
 194                         'secret_mounts': {},
 195                         'output_storage_classes': ["default"]
 196                     }))
 197
 198     # The test passes some fields in builder.resources
 199     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 200     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 201     def test_resource_requirements(self, keepdocker):
 202         arvados_cwl.add_arv_hints()
 203         runner = mock.MagicMock()
 204         runner.ignore_docker_for_reuse = False
 205         runner.intermediate_output_ttl = 3600
 206         runner.secret_store = cwltool.secrets.SecretStore()
 207         runner.api._rootDesc = {"revision": "20210628"}
 208         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 209
 210         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 211         runner.api.collections().get().execute.return_value = {
 212             "portable_data_hash": "99999999999999999999999999999993+99"}
 213
 214         tool = cmap({
 215             "inputs": [],
 216             "outputs": [],
 217             "hints": [{
 218                 "class": "ResourceRequirement",
 219                 "coresMin": 3,
 220                 "ramMin": 3000,
 221                 "tmpdirMin": 4000,
 222                 "outdirMin": 5000
 223             }, {
 224                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 225                 "keep_cache": 512
 226             }, {
 227                 "class": "http://arvados.org/cwl#APIRequirement",
 228             }, {
 229                 "class": "http://arvados.org/cwl#PartitionRequirement",
 230                 "partition": "blurb"
 231             }, {
 232                 "class": "http://arvados.org/cwl#IntermediateOutput",
 233                 "outputTTL": 7200
 234             }, {
 235                 "class": "WorkReuse",
 236                 "enableReuse": False
 237             }],
 238             "baseCommand": "ls",
 239             "id": "",
 240             "class": "CommandLineTool",
 241             "cwlVersion": "v1.2"
 242         })
 243
 244         loadingContext, runtimeContext = self.helper(runner)
 245         runtimeContext.name = "test_resource_requirements"
 246
 247         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 248         arvtool.formatgraph = None
 249         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 250             j.run(runtimeContext)
 251
 252         call_args, call_kwargs = runner.api.container_requests().create.call_args
 253
 254         call_body_expected = {
 255             'environment': {
 256                 'HOME': '/var/spool/cwl',
 257                 'TMPDIR': '/tmp'
 258             },
 259             'name': 'test_resource_requirements',
 260             'runtime_constraints': {
 261                 'vcpus': 3,
 262                 'ram': 3145728000,
 263                 'keep_cache_ram': 536870912,
 264                 'API': True
 265             },
 266             'use_existing': False,
 267             'priority': 500,
 268             'mounts': {
 269                 '/tmp': {'kind': 'tmp',
 270                          "capacity": 4194304000 },
 271                 '/var/spool/cwl': {'kind': 'tmp',
 272                                    "capacity": 5242880000 }
 273             },
 274             'state': 'Committed',
 275             'output_name': 'Output from step test_resource_requirements',
 276             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 277             'output_path': '/var/spool/cwl',
 278             'output_ttl': 7200,
 279             'container_image': '99999999999999999999999999999993+99',
 280             'command': ['ls'],
 281             'cwd': '/var/spool/cwl',
 282             'scheduling_parameters': {
 283                 'partitions': ['blurb']
 284             },
 285             'properties': {'cwl_input': {}},
 286             'secret_mounts': {},
 287             'output_storage_classes': ["default"]
 288         }
 289
 290         call_body = call_kwargs.get('body', None)
 291         self.assertNotEqual(None, call_body)
 292         for key in call_body:
 293             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 294
 295
 296     # The test passes some fields in builder.resources
 297     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 298     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 299     @mock.patch("arvados.collection.Collection")
 300     def test_initial_work_dir(self, collection_mock, keepdocker):
 301         runner = mock.MagicMock()
 302         runner.ignore_docker_for_reuse = False
 303         runner.intermediate_output_ttl = 0
 304         runner.secret_store = cwltool.secrets.SecretStore()
 305         runner.api._rootDesc = {"revision": "20210628"}
 306         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 307
 308         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 309         runner.api.collections().get().execute.return_value = {
 310             "portable_data_hash": "99999999999999999999999999999993+99"}
 311
 312         sourcemock = mock.MagicMock()
 313         def get_collection_mock(p):
 314             if "/" in p:
 315                 return (sourcemock, p.split("/", 1)[1])
 316             else:
 317                 return (sourcemock, "")
 318         runner.fs_access.get_collection.side_effect = get_collection_mock
 319
 320         vwdmock = mock.MagicMock()
 321         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 322
 323         tool = cmap({
 324             "inputs": [],
 325             "outputs": [],
 326             "hints": [{
 327                 "class": "InitialWorkDirRequirement",
 328                 "listing": [{
 329                     "class": "File",
 330                     "basename": "foo",
 331                     "location": "keep:99999999999999999999999999999995+99/bar"
 332                 },
 333                 {
 334                     "class": "Directory",
 335                     "basename": "foo2",
 336                     "location": "keep:99999999999999999999999999999995+99"
 337                 },
 338                 {
 339                     "class": "File",
 340                     "basename": "filename",
 341                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 342                 },
 343                 {
 344                     "class": "Directory",
 345                     "basename": "subdir",
 346                     "location": "keep:99999999999999999999999999999995+99/subdir"
 347                 }                        ]
 348             }],
 349             "baseCommand": "ls",
 350             "class": "CommandLineTool",
 351             "cwlVersion": "v1.2",
 352             "id": ""
 353         })
 354
 355         loadingContext, runtimeContext = self.helper(runner)
 356         runtimeContext.name = "test_initial_work_dir"
 357
 358         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 359
 360         arvtool.formatgraph = None
 361         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 362             j.run(runtimeContext)
 363
 364         call_args, call_kwargs = runner.api.container_requests().create.call_args
 365
 366         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 367         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 368         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 369         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 370
 371         call_body_expected = {
 372             'environment': {
 373                 'HOME': '/var/spool/cwl',
 374                 'TMPDIR': '/tmp'
 375             },
 376             'name': 'test_initial_work_dir',
 377             'runtime_constraints': {
 378                 'vcpus': 1,
 379                 'ram': 268435456
 380             },
 381             'use_existing': True,
 382             'priority': 500,
 383             'mounts': {
 384                 '/tmp': {'kind': 'tmp',
 385                          "capacity": 1073741824 },
 386                 '/var/spool/cwl': {'kind': 'tmp',
 387                                    "capacity": 1073741824 },
 388                 '/var/spool/cwl/foo': {
 389                     'kind': 'collection',
 390                     'path': 'foo',
 391                     'portable_data_hash': '99999999999999999999999999999996+99'
 392                 },
 393                 '/var/spool/cwl/foo2': {
 394                     'kind': 'collection',
 395                     'path': 'foo2',
 396                     'portable_data_hash': '99999999999999999999999999999996+99'
 397                 },
 398                 '/var/spool/cwl/filename': {
 399                     'kind': 'collection',
 400                     'path': 'filename',
 401                     'portable_data_hash': '99999999999999999999999999999996+99'
 402                 },
 403                 '/var/spool/cwl/subdir': {
 404                     'kind': 'collection',
 405                     'path': 'subdir',
 406                     'portable_data_hash': '99999999999999999999999999999996+99'
 407                 }
 408             },
 409             'state': 'Committed',
 410             'output_name': 'Output from step test_initial_work_dir',
 411             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 412             'output_path': '/var/spool/cwl',
 413             'output_ttl': 0,
 414             'container_image': '99999999999999999999999999999993+99',
 415             'command': ['ls'],
 416             'cwd': '/var/spool/cwl',
 417             'scheduling_parameters': {
 418             },
 419             'properties': {'cwl_input': {}},
 420             'secret_mounts': {},
 421             'output_storage_classes': ["default"]
 422         }
 423
 424         call_body = call_kwargs.get('body', None)
 425         self.assertNotEqual(None, call_body)
 426         for key in call_body:
 427             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 428
 429
 430     # Test redirecting stdin/stdout/stderr
 431     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 432     def test_redirects(self, keepdocker):
 433         runner = mock.MagicMock()
 434         runner.ignore_docker_for_reuse = False
 435         runner.intermediate_output_ttl = 0
 436         runner.secret_store = cwltool.secrets.SecretStore()
 437         runner.api._rootDesc = {"revision": "20210628"}
 438         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 439
 440         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 441         runner.api.collections().get().execute.return_value = {
 442             "portable_data_hash": "99999999999999999999999999999993+99"}
 443
 444         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 445
 446         tool = cmap({
 447             "inputs": [],
 448             "outputs": [],
 449             "baseCommand": "ls",
 450             "stdout": "stdout.txt",
 451             "stderr": "stderr.txt",
 452             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 453             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 454             "id": "",
 455             "class": "CommandLineTool",
 456             "cwlVersion": "v1.2"
 457         })
 458
 459         loadingContext, runtimeContext = self.helper(runner)
 460         runtimeContext.name = "test_run_redirect"
 461
 462         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 463         arvtool.formatgraph = None
 464         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 465             j.run(runtimeContext)
 466             runner.api.container_requests().create.assert_called_with(
 467                 body=JsonDiffMatcher({
 468                     'environment': {
 469                         'HOME': '/var/spool/cwl',
 470                         'TMPDIR': '/tmp'
 471                     },
 472                     'name': 'test_run_redirect',
 473                     'runtime_constraints': {
 474                         'vcpus': 1,
 475                         'ram': 268435456
 476                     },
 477                     'use_existing': True,
 478                     'priority': 500,
 479                     'mounts': {
 480                         '/tmp': {'kind': 'tmp',
 481                                  "capacity": 1073741824 },
 482                         '/var/spool/cwl': {'kind': 'tmp',
 483                                            "capacity": 1073741824 },
 484                         "stderr": {
 485                             "kind": "file",
 486                             "path": "/var/spool/cwl/stderr.txt"
 487                         },
 488                         "stdin": {
 489                             "kind": "collection",
 490                             "path": "file.txt",
 491                             "portable_data_hash": "99999999999999999999999999999996+99"
 492                         },
 493                         "stdout": {
 494                             "kind": "file",
 495                             "path": "/var/spool/cwl/stdout.txt"
 496                         },
 497                     },
 498                     'state': 'Committed',
 499                     "output_name": "Output from step test_run_redirect",
 500                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 501                     'output_path': '/var/spool/cwl',
 502                     'output_ttl': 0,
 503                     'container_image': '99999999999999999999999999999993+99',
 504                     'command': ['ls', '/var/spool/cwl'],
 505                     'cwd': '/var/spool/cwl',
 506                     'scheduling_parameters': {},
 507                     'properties': {'cwl_input': {}},
 508                     'secret_mounts': {},
 509                     'output_storage_classes': ["default"]
 510                 }))
 511
 512     @mock.patch("arvados.collection.Collection")
 513     def test_done(self, col):
 514         api = mock.MagicMock()
 515
 516         runner = mock.MagicMock()
 517         runner.api = api
 518         runner.num_retries = 0
 519         runner.ignore_docker_for_reuse = False
 520         runner.intermediate_output_ttl = 0
 521         runner.secret_store = cwltool.secrets.SecretStore()
 522
 523         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 524
 525         runner.api.containers().get().execute.return_value = {"state":"Complete",
 526                                                               "output": "abc+123",
 527                                                               "exit_code": 0}
 528
 529         # Need to noop-out the close method otherwise it gets
 530         # discarded when closed and we can't call getvalue() to check
 531         # it.
 532         class NoopCloseStringIO(io.StringIO):
 533             def close(self):
 534                 pass
 535
 536         usage_report = NoopCloseStringIO()
 537         def colreader_action(name, mode):
 538             nonlocal usage_report
 539             if name == "node.json":
 540                 return io.StringIO("""{
 541     "ProviderType": "c5.large",
 542     "VCPUs": 2,
 543     "RAM": 4294967296,
 544     "IncludedScratch": 8000000000000,
 545     "AddedScratch": 0,
 546     "Price": 0.085,
 547     "Preemptible": false,
 548     "CUDA": {
 549         "DriverVersion": "",
 550         "HardwareCapability": "",
 551         "DeviceCount": 0
 552     }
 553 }""")
 554             if name == 'crunchstat.txt':
 555                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 556             if name == 'arv-mount.txt':
 557                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 558             if name == 'usage_report.html':
 559                 return usage_report
 560             return None
 561
 562         col().open.side_effect = colreader_action
 563         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 564
 565         loadingContext, runtimeContext = self.helper(runner)
 566
 567         arvjob = arvados_cwl.ArvadosContainer(runner,
 568                                               runtimeContext,
 569                                               mock.MagicMock(),
 570                                               {},
 571                                               None,
 572                                               [],
 573                                               [],
 574                                               "testjob")
 575         arvjob.output_callback = mock.MagicMock()
 576         arvjob.collect_outputs = mock.MagicMock()
 577         arvjob.successCodes = [0]
 578         arvjob.outdir = "/var/spool/cwl"
 579         arvjob.output_ttl = 3600
 580         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 581
 582         arvjob.collect_outputs.return_value = {"out": "stuff"}
 583
 584         arvjob.done({
 585             "state": "Final",
 586             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 587             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 588             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 589             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 590             "modified_at": "2017-05-26T12:01:22Z",
 591             "properties": {},
 592             "name": "testjob"
 593         })
 594
 595         self.assertFalse(api.collections().create.called)
 596         self.assertFalse(runner.runtime_status_error.called)
 597
 598         # Assert that something was written to the usage report
 599         self.assertTrue(len(usage_report.getvalue()) > 0)
 600
 601         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 602         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 603         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 604
 605         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 606                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 607
 608
 609     # Test to make sure we dont call runtime_status_update if we already did
 610     # some where higher up in the call stack
 611     @mock.patch("arvados_cwl.util.get_current_container")
 612     def test_recursive_runtime_status_update(self, gcc_mock):
 613         self.setup_and_test_container_executor_and_logging(gcc_mock)
 614         root_logger = logging.getLogger('')
 615
 616         # get_current_container is invoked when we call runtime_status_update
 617         # so try and log again!
 618         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 619         try:
 620             root_logger.error("First Error")
 621         except RuntimeError:
 622             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 623
 624
 625     # Test to make sure that an exception raised from
 626     # get_current_container doesn't cause the logger to raise an
 627     # exception
 628     @mock.patch("arvados_cwl.util.get_current_container")
 629     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 630         self.setup_and_test_container_executor_and_logging(gcc_mock)
 631         root_logger = logging.getLogger('')
 632
 633         # get_current_container is invoked when we call
 634         # runtime_status_update, it is going to also raise an
 635         # exception.
 636         gcc_mock.side_effect = Exception("Second Error")
 637         try:
 638             root_logger.error("First Error")
 639         except Exception:
 640             self.fail("Exception in logger should not propagate")
 641         self.assertTrue(gcc_mock.called)
 642
 643     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 644     @mock.patch("arvados_cwl.util.get_current_container")
 645     @mock.patch("arvados.collection.CollectionReader")
 646     @mock.patch("arvados.collection.Collection")
 647     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 648         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 649
 650         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 651         self.assertTrue(gcc_mock.called)
 652
 653         runner.num_retries = 0
 654         runner.ignore_docker_for_reuse = False
 655         runner.intermediate_output_ttl = 0
 656         runner.secret_store = cwltool.secrets.SecretStore()
 657         runner.label = mock.MagicMock()
 658         runner.label.return_value = '[container testjob]'
 659
 660         runner.api.containers().get().execute.return_value = {
 661             "state":"Complete",
 662             "output": "abc+123",
 663             "exit_code": 1,
 664             "log": "def+234"
 665         }
 666
 667         col().open.return_value = []
 668
 669         loadingContext, runtimeContext = self.helper(runner)
 670
 671         arvjob = arvados_cwl.ArvadosContainer(runner,
 672                                               runtimeContext,
 673                                               mock.MagicMock(),
 674                                               {},
 675                                               None,
 676                                               [],
 677                                               [],
 678                                               "testjob")
 679         arvjob.output_callback = mock.MagicMock()
 680         arvjob.collect_outputs = mock.MagicMock()
 681         arvjob.successCodes = [0]
 682         arvjob.outdir = "/var/spool/cwl"
 683         arvjob.output_ttl = 3600
 684         arvjob.collect_outputs.return_value = {"out": "stuff"}
 685
 686         arvjob.done({
 687             "state": "Final",
 688             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 689             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 690             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 691             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 692             "modified_at": "2017-05-26T12:01:22Z",
 693             "properties": {}
 694         })
 695
 696         rts_mock.assert_has_calls([
 697             mock.call('error',
 698                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 699                       '  ** log is empty **'
 700                       ),
 701             mock.call('warning',
 702                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 703         )])
 704         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 705
 706     # The test passes no builder.resources
 707     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 708     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 709     def test_mounts(self, keepdocker):
 710         runner = mock.MagicMock()
 711         runner.ignore_docker_for_reuse = False
 712         runner.intermediate_output_ttl = 0
 713         runner.secret_store = cwltool.secrets.SecretStore()
 714         runner.api._rootDesc = {"revision": "20210628"}
 715         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 716
 717         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 718         runner.api.collections().get().execute.return_value = {
 719             "portable_data_hash": "99999999999999999999999999999994+99",
 720             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 721
 722         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 723
 724         tool = cmap({
 725             "inputs": [
 726                 {"id": "p1",
 727                  "type": "Directory"}
 728             ],
 729             "outputs": [],
 730             "baseCommand": "ls",
 731             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 732             "id": "",
 733             "class": "CommandLineTool",
 734             "cwlVersion": "v1.2"
 735         })
 736
 737         loadingContext, runtimeContext = self.helper(runner)
 738         runtimeContext.name = "test_run_mounts"
 739
 740         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 741         arvtool.formatgraph = None
 742         job_order = {
 743             "p1": {
 744                 "class": "Directory",
 745                 "location": "keep:99999999999999999999999999999994+44",
 746                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 747                 "listing": [
 748                     {
 749                         "class": "File",
 750                         "location": "keep:99999999999999999999999999999994+44/file1",
 751                     },
 752                     {
 753                         "class": "File",
 754                         "location": "keep:99999999999999999999999999999994+44/file2",
 755                     }
 756                 ]
 757             }
 758         }
 759         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 760             j.run(runtimeContext)
 761             runner.api.container_requests().create.assert_called_with(
 762                 body=JsonDiffMatcher({
 763                     'environment': {
 764                         'HOME': '/var/spool/cwl',
 765                         'TMPDIR': '/tmp'
 766                     },
 767                     'name': 'test_run_mounts',
 768                     'runtime_constraints': {
 769                         'vcpus': 1,
 770                         'ram': 268435456
 771                     },
 772                     'use_existing': True,
 773                     'priority': 500,
 774                     'mounts': {
 775                         "/keep/99999999999999999999999999999994+44": {
 776                             "kind": "collection",
 777                             "portable_data_hash": "99999999999999999999999999999994+44",
 778                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 779                         },
 780                         '/tmp': {'kind': 'tmp',
 781                                  "capacity": 1073741824 },
 782                         '/var/spool/cwl': {'kind': 'tmp',
 783                                            "capacity": 1073741824 }
 784                     },
 785                     'state': 'Committed',
 786                     'output_name': 'Output from step test_run_mounts',
 787                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 788                     'output_path': '/var/spool/cwl',
 789                     'output_ttl': 0,
 790                     'container_image': '99999999999999999999999999999994+99',
 791                     'command': ['ls', '/var/spool/cwl'],
 792                     'cwd': '/var/spool/cwl',
 793                     'scheduling_parameters': {},
 794                     'properties': {'cwl_input': {
 795                         "p1": {
 796                             "basename": "99999999999999999999999999999994+44",
 797                             "class": "Directory",
 798                             "dirname": "/keep",
 799                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 800                             "listing": [
 801                                 {
 802                                     "basename": "file1",
 803                                     "class": "File",
 804                                     "dirname": "/keep/99999999999999999999999999999994+44",
 805                                     "location": "keep:99999999999999999999999999999994+44/file1",
 806                                     "nameext": "",
 807                                     "nameroot": "file1",
 808                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 809                                     "size": 0
 810                                 },
 811                                 {
 812                                     "basename": "file2",
 813                                     "class": "File",
 814                                     "dirname": "/keep/99999999999999999999999999999994+44",
 815                                     "location": "keep:99999999999999999999999999999994+44/file2",
 816                                     "nameext": "",
 817                                     "nameroot": "file2",
 818                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 819                                     "size": 0
 820                                 }
 821                             ],
 822                             "location": "keep:99999999999999999999999999999994+44",
 823                             "path": "/keep/99999999999999999999999999999994+44"
 824                         }
 825                     }},
 826                     'secret_mounts': {},
 827                     'output_storage_classes': ["default"]
 828                 }))
 829
 830     # The test passes no builder.resources
 831     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 832     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 833     def test_secrets(self, keepdocker):
 834         arvados_cwl.add_arv_hints()
 835         runner = mock.MagicMock()
 836         runner.ignore_docker_for_reuse = False
 837         runner.intermediate_output_ttl = 0
 838         runner.secret_store = cwltool.secrets.SecretStore()
 839         runner.api._rootDesc = {"revision": "20210628"}
 840         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 841
 842         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 843         runner.api.collections().get().execute.return_value = {
 844             "portable_data_hash": "99999999999999999999999999999993+99"}
 845
 846         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 847
 848         tool = cmap({"arguments": ["md5sum", "example.conf"],
 849                      "class": "CommandLineTool",
 850                      "cwlVersion": "v1.2",
 851                      "hints": [
 852                          {
 853                              "class": "http://commonwl.org/cwltool#Secrets",
 854                              "secrets": [
 855                                  "#secret_job.cwl/pw"
 856                              ]
 857                          }
 858                      ],
 859                      "id": "",
 860                      "inputs": [
 861                          {
 862                              "id": "#secret_job.cwl/pw",
 863                              "type": "string"
 864                          }
 865                      ],
 866                      "outputs": [
 867                      ],
 868                      "requirements": [
 869                          {
 870                              "class": "InitialWorkDirRequirement",
 871                              "listing": [
 872                                  {
 873                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 874                                      "entryname": "example.conf"
 875                                  }
 876                              ]
 877                          }
 878                      ]})
 879
 880         loadingContext, runtimeContext = self.helper(runner)
 881         runtimeContext.name = "test_secrets"
 882
 883         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 884         arvtool.formatgraph = None
 885
 886         job_order = {"pw": "blorp"}
 887         runner.secret_store.store(["pw"], job_order)
 888
 889         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 890             j.run(runtimeContext)
 891             runner.api.container_requests().create.assert_called_with(
 892                 body=JsonDiffMatcher({
 893                     'environment': {
 894                         'HOME': '/var/spool/cwl',
 895                         'TMPDIR': '/tmp'
 896                     },
 897                     'name': 'test_secrets',
 898                     'runtime_constraints': {
 899                         'vcpus': 1,
 900                         'ram': 268435456
 901                     },
 902                     'use_existing': True,
 903                     'priority': 500,
 904                     'mounts': {
 905                         '/tmp': {'kind': 'tmp',
 906                                  "capacity": 1073741824
 907                              },
 908                         '/var/spool/cwl': {'kind': 'tmp',
 909                                            "capacity": 1073741824 }
 910                     },
 911                     'state': 'Committed',
 912                     'output_name': 'Output from step test_secrets',
 913                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 914                     'output_path': '/var/spool/cwl',
 915                     'output_ttl': 0,
 916                     'container_image': '99999999999999999999999999999993+99',
 917                     'command': ['md5sum', 'example.conf'],
 918                     'cwd': '/var/spool/cwl',
 919                     'scheduling_parameters': {},
 920                     'properties': {'cwl_input': job_order},
 921                     "secret_mounts": {
 922                         "/var/spool/cwl/example.conf": {
 923                             "content": "username: user\npassword: blorp\n",
 924                             "kind": "text"
 925                         }
 926                     },
 927                     'output_storage_classes': ["default"]
 928                 }))
 929
 930     # The test passes no builder.resources
 931     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 932     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 933     def test_timelimit(self, keepdocker):
 934         runner = mock.MagicMock()
 935         runner.ignore_docker_for_reuse = False
 936         runner.intermediate_output_ttl = 0
 937         runner.secret_store = cwltool.secrets.SecretStore()
 938         runner.api._rootDesc = {"revision": "20210628"}
 939         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 940
 941         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 942         runner.api.collections().get().execute.return_value = {
 943             "portable_data_hash": "99999999999999999999999999999993+99"}
 944
 945         tool = cmap({
 946             "inputs": [],
 947             "outputs": [],
 948             "baseCommand": "ls",
 949             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 950             "id": "",
 951             "cwlVersion": "v1.2",
 952             "class": "CommandLineTool",
 953             "hints": [
 954                 {
 955                     "class": "ToolTimeLimit",
 956                     "timelimit": 42
 957                 }
 958             ]
 959         })
 960
 961         loadingContext, runtimeContext = self.helper(runner)
 962         runtimeContext.name = "test_timelimit"
 963
 964         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 965         arvtool.formatgraph = None
 966
 967         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 968             j.run(runtimeContext)
 969
 970         _, kwargs = runner.api.container_requests().create.call_args
 971         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 972
 973
 974     # The test passes no builder.resources
 975     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 976     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 977     def test_setting_storage_class(self, keepdocker):
 978         #arv_docker_clear_cache()
 979
 980         runner = mock.MagicMock()
 981         runner.ignore_docker_for_reuse = False
 982         runner.intermediate_output_ttl = 0
 983         runner.secret_store = cwltool.secrets.SecretStore()
 984         runner.api._rootDesc = {"revision": "20210628"}
 985         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 986
 987         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 988         runner.api.collections().get().execute.return_value = {
 989             "portable_data_hash": "99999999999999999999999999999993+99"}
 990
 991         tool = cmap({
 992             "inputs": [],
 993             "outputs": [],
 994             "baseCommand": "ls",
 995             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 996             "id": "",
 997             "cwlVersion": "v1.2",
 998             "class": "CommandLineTool",
 999             "hints": [
1000                 {
1001                     "class": "http://arvados.org/cwl#OutputStorageClass",
1002                     "finalStorageClass": ["baz_sc", "qux_sc"],
1003                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1004                 }
1005             ]
1006         })
1007
1008         loadingContext, runtimeContext = self.helper(runner, True)
1009
1010         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1011         arvtool.formatgraph = None
1012
1013         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1014             j.run(runtimeContext)
1015             runner.api.container_requests().create.assert_called_with(
1016                 body=JsonDiffMatcher({
1017                     'environment': {
1018                         'HOME': '/var/spool/cwl',
1019                         'TMPDIR': '/tmp'
1020                     },
1021                     'name': 'test_run_True',
1022                     'runtime_constraints': {
1023                         'vcpus': 1,
1024                         'ram': 268435456
1025                     },
1026                     'use_existing': True,
1027                     'priority': 500,
1028                     'mounts': {
1029                         '/tmp': {'kind': 'tmp',
1030                                  "capacity": 1073741824
1031                              },
1032                         '/var/spool/cwl': {'kind': 'tmp',
1033                                            "capacity": 1073741824 }
1034                     },
1035                     'state': 'Committed',
1036                     'output_name': 'Output from step test_run_True',
1037                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1038                     'output_path': '/var/spool/cwl',
1039                     'output_ttl': 0,
1040                     'container_image': '99999999999999999999999999999993+99',
1041                     'command': ['ls', '/var/spool/cwl'],
1042                     'cwd': '/var/spool/cwl',
1043                     'scheduling_parameters': {},
1044                     'properties': {'cwl_input': {}},
1045                     'secret_mounts': {},
1046                     'output_storage_classes': ["foo_sc", "bar_sc"]
1047                 }))
1048
1049
1050     # The test passes no builder.resources
1051     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1052     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1053     def test_setting_process_properties(self, keepdocker):
1054         #arv_docker_clear_cache()
1055
1056         runner = mock.MagicMock()
1057         runner.ignore_docker_for_reuse = False
1058         runner.intermediate_output_ttl = 0
1059         runner.secret_store = cwltool.secrets.SecretStore()
1060         runner.api._rootDesc = {"revision": "20210628"}
1061         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1062
1063         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1064         runner.api.collections().get().execute.return_value = {
1065             "portable_data_hash": "99999999999999999999999999999993+99"}
1066
1067         tool = cmap({
1068             "inputs": [
1069                 {"id": "x", "type": "string"}],
1070             "outputs": [],
1071             "baseCommand": "ls",
1072             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1073             "id": "",
1074             "class": "CommandLineTool",
1075             "cwlVersion": "v1.2",
1076             "hints": [
1077             {
1078                 "class": "http://arvados.org/cwl#ProcessProperties",
1079                 "processProperties": [
1080                     {"propertyName": "foo",
1081                      "propertyValue": "bar"},
1082                     {"propertyName": "baz",
1083                      "propertyValue": "$(inputs.x)"},
1084                     {"propertyName": "quux",
1085                      "propertyValue": {
1086                          "q1": 1,
1087                          "q2": 2
1088                      }
1089                     }
1090                 ],
1091             }
1092         ]
1093         })
1094
1095         loadingContext, runtimeContext = self.helper(runner, True)
1096
1097         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1098         arvtool.formatgraph = None
1099
1100         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1101             j.run(runtimeContext)
1102             runner.api.container_requests().create.assert_called_with(
1103                 body=JsonDiffMatcher({
1104                     'environment': {
1105                         'HOME': '/var/spool/cwl',
1106                         'TMPDIR': '/tmp'
1107                     },
1108                     'name': 'test_run_True',
1109                     'runtime_constraints': {
1110                         'vcpus': 1,
1111                         'ram': 268435456
1112                     },
1113                     'use_existing': True,
1114                     'priority': 500,
1115                     'mounts': {
1116                         '/tmp': {'kind': 'tmp',
1117                                  "capacity": 1073741824
1118                              },
1119                         '/var/spool/cwl': {'kind': 'tmp',
1120                                            "capacity": 1073741824 }
1121                     },
1122                     'state': 'Committed',
1123                     'output_name': 'Output from step test_run_True',
1124                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1125                     'output_path': '/var/spool/cwl',
1126                     'output_ttl': 0,
1127                     'container_image': '99999999999999999999999999999993+99',
1128                     'command': ['ls', '/var/spool/cwl'],
1129                     'cwd': '/var/spool/cwl',
1130                     'scheduling_parameters': {},
1131                     'properties': {
1132                         "baz": "blorp",
1133                         "cwl_input": {"x": "blorp"},
1134                         "foo": "bar",
1135                         "quux": {
1136                             "q1": 1,
1137                             "q2": 2
1138                         }
1139                     },
1140                     'secret_mounts': {},
1141                     'output_storage_classes': ["default"]
1142                 }))
1143
1144
1145     # The test passes no builder.resources
1146     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1147     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1148     def test_cuda_requirement(self, keepdocker):
1149         arvados_cwl.add_arv_hints()
1150         #arv_docker_clear_cache()
1151
1152         runner = mock.MagicMock()
1153         runner.ignore_docker_for_reuse = False
1154         runner.intermediate_output_ttl = 0
1155         runner.secret_store = cwltool.secrets.SecretStore()
1156         runner.api._rootDesc = {"revision": "20210628"}
1157         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1158
1159         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1160         runner.api.collections().get().execute.return_value = {
1161             "portable_data_hash": "99999999999999999999999999999993+99"}
1162
1163         test_cwl_req = [{
1164                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1165                 "cudaVersionMin": "11.0",
1166                 "cudaComputeCapability": "9.0",
1167             }, {
1168                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1169                 "cudaVersionMin": "11.0",
1170                 "cudaComputeCapability": "9.0",
1171                 "cudaDeviceCountMin": 2
1172             }, {
1173                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1174                 "cudaVersionMin": "11.0",
1175                 "cudaComputeCapability": ["4.0", "5.0"],
1176                 "cudaDeviceCountMin": 2
1177             }]
1178
1179         test_arv_req = [{
1180             'device_count': 1,
1181             'driver_version': "11.0",
1182             'hardware_capability': "9.0"
1183         }, {
1184             'device_count': 2,
1185             'driver_version': "11.0",
1186             'hardware_capability': "9.0"
1187         }, {
1188             'device_count': 2,
1189             'driver_version': "11.0",
1190             'hardware_capability': "4.0"
1191         }]
1192
1193         for test_case in range(0, len(test_cwl_req)):
1194
1195             tool = cmap({
1196                 "inputs": [],
1197                 "outputs": [],
1198                 "baseCommand": "nvidia-smi",
1199                 "arguments": [],
1200                 "id": "",
1201                 "cwlVersion": "v1.2",
1202                 "class": "CommandLineTool",
1203                 "requirements": [test_cwl_req[test_case]]
1204             })
1205
1206             loadingContext, runtimeContext = self.helper(runner, True)
1207
1208             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1209             arvtool.formatgraph = None
1210
1211             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1212                 j.run(runtimeContext)
1213                 runner.api.container_requests().create.assert_called_with(
1214                     body=JsonDiffMatcher({
1215                         'environment': {
1216                             'HOME': '/var/spool/cwl',
1217                             'TMPDIR': '/tmp'
1218                         },
1219                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1220                         'runtime_constraints': {
1221                             'vcpus': 1,
1222                             'ram': 268435456,
1223                             'cuda': test_arv_req[test_case]
1224                         },
1225                         'use_existing': True,
1226                         'priority': 500,
1227                         'mounts': {
1228                             '/tmp': {'kind': 'tmp',
1229                                      "capacity": 1073741824
1230                                  },
1231                             '/var/spool/cwl': {'kind': 'tmp',
1232                                                "capacity": 1073741824 }
1233                         },
1234                         'state': 'Committed',
1235                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1236                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1237                         'output_path': '/var/spool/cwl',
1238                         'output_ttl': 0,
1239                         'container_image': '99999999999999999999999999999993+99',
1240                         'command': ['nvidia-smi'],
1241                         'cwd': '/var/spool/cwl',
1242                         'scheduling_parameters': {},
1243                         'properties': {'cwl_input': {}},
1244                         'secret_mounts': {},
1245                         'output_storage_classes': ["default"]
1246                     }))
1247
1248
1249     # The test passes no builder.resources
1250     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1251     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1252     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1253     def test_match_local_docker(self, keepdocker, determine_image_id):
1254         arvados_cwl.add_arv_hints()
1255
1256         runner = mock.MagicMock()
1257         runner.ignore_docker_for_reuse = False
1258         runner.intermediate_output_ttl = 0
1259         runner.secret_store = cwltool.secrets.SecretStore()
1260         runner.api._rootDesc = {"revision": "20210628"}
1261         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1262
1263         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1264                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1265         determine_image_id.side_effect = lambda x: "123"
1266         def execute(uuid):
1267             ex = mock.MagicMock()
1268             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1269                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1270             ex.execute.return_value = lookup[uuid]
1271             return ex
1272         runner.api.collections().get.side_effect = execute
1273
1274         tool = cmap({
1275             "inputs": [],
1276             "outputs": [],
1277             "baseCommand": "echo",
1278             "arguments": [],
1279             "id": "",
1280             "cwlVersion": "v1.0",
1281             "class": "org.w3id.cwl.cwl.CommandLineTool"
1282         })
1283
1284         loadingContext, runtimeContext = self.helper(runner, True)
1285
1286         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1287         arvtool.formatgraph = None
1288
1289         container_request = {
1290             'environment': {
1291                 'HOME': '/var/spool/cwl',
1292                 'TMPDIR': '/tmp'
1293             },
1294             'name': 'test_run_True',
1295             'runtime_constraints': {
1296                 'vcpus': 1,
1297                 'ram': 1073741824,
1298             },
1299             'use_existing': True,
1300             'priority': 500,
1301             'mounts': {
1302                 '/tmp': {'kind': 'tmp',
1303                          "capacity": 1073741824
1304                          },
1305                 '/var/spool/cwl': {'kind': 'tmp',
1306                                    "capacity": 1073741824 }
1307             },
1308             'state': 'Committed',
1309             'output_name': 'Output from step test_run_True',
1310             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1311             'output_path': '/var/spool/cwl',
1312             'output_ttl': 0,
1313             'container_image': '99999999999999999999999999999994+99',
1314             'command': ['echo'],
1315             'cwd': '/var/spool/cwl',
1316             'scheduling_parameters': {},
1317             'properties': {'cwl_input': {}},
1318             'secret_mounts': {},
1319             'output_storage_classes': ["default"]
1320         }
1321
1322         runtimeContext.match_local_docker = False
1323         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1324             j.run(runtimeContext)
1325             runner.api.container_requests().create.assert_called_with(
1326                 body=JsonDiffMatcher(container_request))
1327
1328         runtimeContext.cached_docker_lookups.clear()
1329         runtimeContext.match_local_docker = True
1330         container_request['container_image'] = '99999999999999999999999999999993+99'
1331         container_request['name'] = 'test_run_True_2'
1332         container_request['output_name'] = 'Output from step test_run_True_2'
1333         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1334             j.run(runtimeContext)
1335             runner.api.container_requests().create.assert_called_with(
1336                 body=JsonDiffMatcher(container_request))
1337
1338
1339     # The test passes no builder.resources
1340     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1341     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1342     def test_run_preemptible_hint(self, keepdocker):
1343         arvados_cwl.add_arv_hints()
1344         for enable_preemptible in (None, True, False):
1345             for preemptible_hint in (None, True, False):
1346                 #arv_docker_clear_cache()
1347
1348                 runner = mock.MagicMock()
1349                 runner.ignore_docker_for_reuse = False
1350                 runner.intermediate_output_ttl = 0
1351                 runner.secret_store = cwltool.secrets.SecretStore()
1352                 runner.api._rootDesc = {"revision": "20210628"}
1353                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1354
1355                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1356                 runner.api.collections().get().execute.return_value = {
1357                     "portable_data_hash": "99999999999999999999999999999993+99"}
1358
1359                 if preemptible_hint is not None:
1360                     hints = [{
1361                         "class": "http://arvados.org/cwl#UsePreemptible",
1362                         "usePreemptible": preemptible_hint
1363                     }]
1364                 else:
1365                     hints = []
1366
1367                 tool = cmap({
1368                     "inputs": [],
1369                     "outputs": [],
1370                     "baseCommand": "ls",
1371                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1372                     "id": "",
1373                     "class": "CommandLineTool",
1374                     "cwlVersion": "v1.2",
1375                     "hints": hints
1376                 })
1377
1378                 loadingContext, runtimeContext = self.helper(runner)
1379
1380                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1381                 runtimeContext.enable_preemptible = enable_preemptible
1382
1383                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1384                 arvtool.formatgraph = None
1385
1386                 # Test the interactions between --enable/disable-preemptible
1387                 # and UsePreemptible hint
1388
1389                 if enable_preemptible is None:
1390                     if preemptible_hint is None:
1391                         sched = {}
1392                     else:
1393                         sched = {'preemptible': preemptible_hint}
1394                 else:
1395                     if preemptible_hint is None:
1396                         sched = {'preemptible': enable_preemptible}
1397                     else:
1398                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1399
1400                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1401                     j.run(runtimeContext)
1402                     runner.api.container_requests().create.assert_called_with(
1403                         body=JsonDiffMatcher({
1404                             'environment': {
1405                                 'HOME': '/var/spool/cwl',
1406                                 'TMPDIR': '/tmp'
1407                             },
1408                             'name': runtimeContext.name,
1409                             'runtime_constraints': {
1410                                 'vcpus': 1,
1411                                 'ram': 268435456
1412                             },
1413                             'use_existing': True,
1414                             'priority': 500,
1415                             'mounts': {
1416                                 '/tmp': {'kind': 'tmp',
1417                                          "capacity": 1073741824
1418                                      },
1419                                 '/var/spool/cwl': {'kind': 'tmp',
1420                                                    "capacity": 1073741824 }
1421                             },
1422                             'state': 'Committed',
1423                             'output_name': 'Output from step '+runtimeContext.name,
1424                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1425                             'output_path': '/var/spool/cwl',
1426                             'output_ttl': 0,
1427                             'container_image': '99999999999999999999999999999993+99',
1428                             'command': ['ls', '/var/spool/cwl'],
1429                             'cwd': '/var/spool/cwl',
1430                             'scheduling_parameters': sched,
1431                             'properties': {'cwl_input': {}},
1432                             'secret_mounts': {},
1433                             'output_storage_classes': ["default"]
1434                         }))
1435
1436
1437     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1438     def test_output_properties(self, keepdocker):
1439         arvados_cwl.add_arv_hints()
1440         for rev in ["20210628", "20220510"]:
1441             runner = mock.MagicMock()
1442             runner.ignore_docker_for_reuse = False
1443             runner.intermediate_output_ttl = 0
1444             runner.secret_store = cwltool.secrets.SecretStore()
1445             runner.api._rootDesc = {"revision": rev}
1446             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1447
1448             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1449             runner.api.collections().get().execute.return_value = {
1450                 "portable_data_hash": "99999999999999999999999999999993+99"}
1451
1452             tool = cmap({
1453                 "inputs": [{
1454                     "id": "inp",
1455                     "type": "string"
1456                 }],
1457                 "outputs": [],
1458                 "baseCommand": "ls",
1459                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1460                 "id": "",
1461                 "cwlVersion": "v1.2",
1462                 "class": "CommandLineTool",
1463                 "hints": [
1464                     {
1465                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1466                         "outputProperties": {
1467                             "foo": "bar",
1468                             "baz": "$(inputs.inp)"
1469                         }
1470                     }
1471                 ]
1472             })
1473
1474             loadingContext, runtimeContext = self.helper(runner)
1475             runtimeContext.name = "test_timelimit"
1476
1477             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1478             arvtool.formatgraph = None
1479
1480             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1481                 j.run(runtimeContext)
1482
1483             _, kwargs = runner.api.container_requests().create.call_args
1484             if rev == "20220510":
1485                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1486             else:
1487                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1488
1489
1490 class TestWorkflow(unittest.TestCase):
1491     def setUp(self):
1492         cwltool.process._names = set()
1493         #arv_docker_clear_cache()
1494
1495     def helper(self, runner, enable_reuse=True):
1496         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1497
1498         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1499                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1500
1501         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1502         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1503         document_loader.fetch_text = document_loader.fetcher.fetch_text
1504         document_loader.check_exists = document_loader.fetcher.check_exists
1505
1506         loadingContext = arvados_cwl.context.ArvLoadingContext(
1507             {"avsc_names": avsc_names,
1508              "basedir": "",
1509              "make_fs_access": make_fs_access,
1510              "loader": document_loader,
1511              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1512              "construct_tool_object": runner.arv_make_tool,
1513              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
1514         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1515             {"work_api": "containers",
1516              "basedir": "",
1517              "name": "test_run_wf_"+str(enable_reuse),
1518              "make_fs_access": make_fs_access,
1519              "tmpdir": "/tmp",
1520              "enable_reuse": enable_reuse,
1521              "priority": 500})
1522
1523         return loadingContext, runtimeContext
1524
1525     # The test passes no builder.resources
1526     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1527     @mock.patch("arvados.collection.CollectionReader")
1528     @mock.patch("arvados.collection.Collection")
1529     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1530     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1531         arvados_cwl.add_arv_hints()
1532
1533         api = mock.MagicMock()
1534         api._rootDesc = get_rootDesc()
1535         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1536
1537         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1538         self.assertEqual(runner.work_api, 'containers')
1539
1540         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1541         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1542         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1543                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1544
1545         runner.api.containers().current().execute.return_value = {}
1546
1547         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1548         runner.ignore_docker_for_reuse = False
1549         runner.num_retries = 0
1550         runner.secret_store = cwltool.secrets.SecretStore()
1551
1552         loadingContext, runtimeContext = self.helper(runner)
1553         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1554
1555         mockcollectionreader().exists.return_value = True
1556
1557         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1558         metadata["cwlVersion"] = tool["cwlVersion"]
1559
1560         mockc = mock.MagicMock()
1561         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1562         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1563
1564         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1565         arvtool.formatgraph = None
1566         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1567
1568         next(it).run(runtimeContext)
1569         next(it).run(runtimeContext)
1570
1571         with open("tests/wf/scatter2_subwf.cwl") as f:
1572             subwf = StripYAMLComments(f.read()).rstrip()
1573
1574         runner.api.container_requests().create.assert_called_with(
1575             body=JsonDiffMatcher({
1576                 "command": [
1577                     "cwltool",
1578                     "--no-container",
1579                     "--move-outputs",
1580                     "--preserve-entire-environment",
1581                     "workflow.cwl",
1582                     "cwl.input.yml"
1583                 ],
1584                 "container_image": "99999999999999999999999999999993+99",
1585                 "cwd": "/var/spool/cwl",
1586                 "environment": {
1587                     "HOME": "/var/spool/cwl",
1588                     "TMPDIR": "/tmp"
1589                 },
1590                 "mounts": {
1591                     "/keep/99999999999999999999999999999999+118": {
1592                         "kind": "collection",
1593                         "portable_data_hash": "99999999999999999999999999999999+118"
1594                     },
1595                     "/tmp": {
1596                         "capacity": 1073741824,
1597                         "kind": "tmp"
1598                     },
1599                     "/var/spool/cwl": {
1600                         "capacity": 1073741824,
1601                         "kind": "tmp"
1602                     },
1603                     "/var/spool/cwl/cwl.input.yml": {
1604                         "kind": "collection",
1605                         "path": "cwl.input.yml",
1606                         "portable_data_hash": "99999999999999999999999999999996+99"
1607                     },
1608                     "/var/spool/cwl/workflow.cwl": {
1609                         "kind": "collection",
1610                         "path": "workflow.cwl",
1611                         "portable_data_hash": "99999999999999999999999999999996+99"
1612                     },
1613                     "stdout": {
1614                         "kind": "file",
1615                         "path": "/var/spool/cwl/cwl.output.json"
1616                     }
1617                 },
1618                 "name": "scatterstep",
1619                 "output_name": "Output from step scatterstep",
1620                 "output_path": "/var/spool/cwl",
1621                 "output_ttl": 0,
1622                 "priority": 500,
1623                 "properties": {'cwl_input': {
1624                         "fileblub": {
1625                             "basename": "token.txt",
1626                             "class": "File",
1627                             "dirname": "/keep/99999999999999999999999999999999+118",
1628                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1629                             "nameext": ".txt",
1630                             "nameroot": "token",
1631                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1632                             "size": 0
1633                         },
1634                         "sleeptime": 5
1635                 }},
1636                 "runtime_constraints": {
1637                     "ram": 1073741824,
1638                     "vcpus": 1
1639                 },
1640                 "scheduling_parameters": {},
1641                 "secret_mounts": {},
1642                 "state": "Committed",
1643                 "use_existing": True,
1644                 'output_storage_classes': ["default"]
1645             }))
1646         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1647         mockc.open().__enter__().write.assert_has_calls([mock.call(
1648 '''{
1649   "fileblub": {
1650     "basename": "token.txt",
1651     "class": "File",
1652     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1653     "size": 0
1654   },
1655   "sleeptime": 5
1656 }''')])
1657
1658     # The test passes no builder.resources
1659     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1660     @mock.patch("arvados.collection.CollectionReader")
1661     @mock.patch("arvados.collection.Collection")
1662     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1663     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1664         arvados_cwl.add_arv_hints()
1665
1666         api = mock.MagicMock()
1667         api._rootDesc = get_rootDesc()
1668         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1669
1670         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1671         self.assertEqual(runner.work_api, 'containers')
1672
1673         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1674         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1675                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1676         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1677                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1678
1679         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1680         runner.ignore_docker_for_reuse = False
1681         runner.num_retries = 0
1682         runner.secret_store = cwltool.secrets.SecretStore()
1683
1684         loadingContext, runtimeContext = self.helper(runner)
1685         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1686         loadingContext.do_update = True
1687         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1688
1689         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1690
1691         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1692         arvtool.formatgraph = None
1693         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1694
1695         next(it).run(runtimeContext)
1696         next(it).run(runtimeContext)
1697
1698         with open("tests/wf/echo-subwf.cwl") as f:
1699             subwf = StripYAMLComments(f.read())
1700
1701         runner.api.container_requests().create.assert_called_with(
1702             body=JsonDiffMatcher({
1703                 'output_ttl': 0,
1704                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1705                 'scheduling_parameters': {},
1706                 'name': u'echo-subwf',
1707                 'secret_mounts': {},
1708                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1709                 'properties': {'cwl_input': {}},
1710                 'priority': 500,
1711                 'mounts': {
1712                     '/var/spool/cwl/cwl.input.yml': {
1713                         'portable_data_hash': '99999999999999999999999999999996+99',
1714                         'kind': 'collection',
1715                         'path': 'cwl.input.yml'
1716                     },
1717                     '/var/spool/cwl/workflow.cwl': {
1718                         'portable_data_hash': '99999999999999999999999999999996+99',
1719                         'kind': 'collection',
1720                         'path': 'workflow.cwl'
1721                     },
1722                     'stdout': {
1723                         'path': '/var/spool/cwl/cwl.output.json',
1724                         'kind': 'file'
1725                     },
1726                     '/tmp': {
1727                         'kind': 'tmp',
1728                         'capacity': 1073741824
1729                     }, '/var/spool/cwl': {
1730                         'kind': 'tmp',
1731                         'capacity': 3221225472
1732                     }
1733                 },
1734                 'state': 'Committed',
1735                 'output_path': '/var/spool/cwl',
1736                 'container_image': '99999999999999999999999999999993+99',
1737                 'command': [
1738                     u'cwltool',
1739                     u'--no-container',
1740                     u'--move-outputs',
1741                     u'--preserve-entire-environment',
1742                     u'workflow.cwl',
1743                     u'cwl.input.yml'
1744                 ],
1745                 'use_existing': True,
1746                 'output_name': u'Output from step echo-subwf',
1747                 'cwd': '/var/spool/cwl',
1748                 'output_storage_classes': ["default"]
1749             }))
1750
1751     def test_default_work_api(self):
1752         arvados_cwl.add_arv_hints()
1753
1754         api = mock.MagicMock()
1755         api._rootDesc = copy.deepcopy(get_rootDesc())
1756         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1757         self.assertEqual(runner.work_api, 'containers')