sdk/cwl/tests/test_job.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import functools
   6 import json
   7 import logging
   8 import mock
   9 import os
  10 import unittest
  11 import copy
  12 import StringIO
  13
  14 import arvados
  15 import arvados_cwl
  16 import cwltool.process
  17 from arvados.errors import ApiError
  18 from schema_salad.ref_resolver import Loader
  19 from schema_salad.sourceline import cmap
  20 from .mock_discovery import get_rootDesc
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .test_container import CollectionMock
  23
  24 if not os.getenv('ARVADOS_DEBUG'):
  25     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  26     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  27
  28 class TestJob(unittest.TestCase):
  29
  30     def helper(self, runner, enable_reuse=True):
  31         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  32
  33         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  34                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  35         loadingContext = arvados_cwl.context.ArvLoadingContext(
  36             {"avsc_names": avsc_names,
  37              "basedir": "",
  38              "make_fs_access": make_fs_access,
  39              "loader": Loader({}),
  40              "metadata": {"cwlVersion": "v1.0"},
  41              "makeTool": runner.arv_make_tool})
  42         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  43             {"work_api": "jobs",
  44              "basedir": "",
  45              "name": "test_run_job_"+str(enable_reuse),
  46              "make_fs_access": make_fs_access,
  47              "enable_reuse": enable_reuse,
  48              "priority": 500})
  49
  50         return loadingContext, runtimeContext
  51
  52     # The test passes no builder.resources
  53     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  54     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
  55     def test_run(self, list_images_in_arv):
  56         for enable_reuse in (True, False):
  57             runner = mock.MagicMock()
  58             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  59             runner.ignore_docker_for_reuse = False
  60             runner.num_retries = 0
  61
  62             list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
  63             runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
  64             # Simulate reused job from another project so that we can check is a can_read
  65             # link is added.
  66             runner.api.jobs().create().execute.return_value = {
  67                 'state': 'Complete' if enable_reuse else 'Queued',
  68                 'owner_uuid': 'zzzzz-tpzed-yyyyyyyyyyyyyyy' if enable_reuse else 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  69                 'uuid': 'zzzzz-819sb-yyyyyyyyyyyyyyy',
  70                 'output': None,
  71             }
  72
  73             tool = cmap({
  74                 "inputs": [],
  75                 "outputs": [],
  76                 "baseCommand": "ls",
  77                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  78                 "id": "#",
  79                 "class": "CommandLineTool"
  80             })
  81
  82             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
  83
  84             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
  85             arvtool.formatgraph = None
  86             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
  87                 j.run(runtimeContext)
  88                 runner.api.jobs().create.assert_called_with(
  89                     body=JsonDiffMatcher({
  90                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  91                         'runtime_constraints': {},
  92                         'script_parameters': {
  93                             'tasks': [{
  94                                 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
  95                                 'command': ['ls', '$(task.outdir)']
  96                             }],
  97                         },
  98                         'script_version': 'master',
  99                         'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 100                         'repository': 'arvados',
 101                         'script': 'crunchrunner',
 102                         'runtime_constraints': {
 103                             'docker_image': 'arvados/jobs',
 104                             'min_cores_per_node': 1,
 105                             'min_ram_mb_per_node': 1024,
 106                             'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
 107                         }
 108                     }),
 109                     find_or_create=enable_reuse,
 110                     filters=[['repository', '=', 'arvados'],
 111                              ['script', '=', 'crunchrunner'],
 112                              ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 113                              ['docker_image_locator', 'in docker', 'arvados/jobs']]
 114                 )
 115                 if enable_reuse:
 116                     runner.api.links().create.assert_called_with(
 117                         body=JsonDiffMatcher({
 118                             'link_class': 'permission',
 119                             'name': 'can_read',
 120                             "tail_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 121                             "head_uuid": "zzzzz-819sb-yyyyyyyyyyyyyyy",
 122                         })
 123                     )
 124                     # Simulate an API excepction when trying to create a
 125                     # sharing link on the job
 126                     runner.api.links().create.side_effect = ApiError(
 127                         mock.MagicMock(return_value={'status': 403}),
 128                         'Permission denied')
 129                     j.run(runtimeContext)
 130                 else:
 131                     assert not runner.api.links().create.called
 132
 133     # The test passes some fields in builder.resources
 134     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 135     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 136     def test_resource_requirements(self, list_images_in_arv):
 137         runner = mock.MagicMock()
 138         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 139         runner.ignore_docker_for_reuse = False
 140         runner.num_retries = 0
 141         arvados_cwl.add_arv_hints()
 142
 143         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 144         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 145
 146         tool = {
 147             "inputs": [],
 148             "outputs": [],
 149             "hints": [{
 150                 "class": "ResourceRequirement",
 151                 "coresMin": 3,
 152                 "ramMin": 3000,
 153                 "tmpdirMin": 4000
 154             }, {
 155                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 156                 "keep_cache": 512,
 157                 "outputDirType": "keep_output_dir"
 158             }, {
 159                 "class": "http://arvados.org/cwl#APIRequirement",
 160             },
 161             {
 162                 "class": "http://arvados.org/cwl#ReuseRequirement",
 163                 "enableReuse": False
 164             }],
 165             "baseCommand": "ls",
 166             "id": "#",
 167             "class": "CommandLineTool"
 168         }
 169
 170         loadingContext, runtimeContext = self.helper(runner)
 171
 172         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 173         arvtool.formatgraph = None
 174         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 175             j.run(runtimeContext)
 176         runner.api.jobs().create.assert_called_with(
 177             body=JsonDiffMatcher({
 178                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 179                 'runtime_constraints': {},
 180                 'script_parameters': {
 181                     'tasks': [{
 182                         'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
 183                         'task.keepTmpOutput': True,
 184                         'command': ['ls']
 185                     }]
 186             },
 187             'script_version': 'master',
 188                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 189                 'repository': 'arvados',
 190                 'script': 'crunchrunner',
 191                 'runtime_constraints': {
 192                     'docker_image': 'arvados/jobs',
 193                     'min_cores_per_node': 3,
 194                     'min_ram_mb_per_node': 3512,     # ramMin + keep_cache
 195                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
 196                     'keep_cache_mb_per_task': 512
 197                 }
 198             }),
 199             find_or_create=False,
 200             filters=[['repository', '=', 'arvados'],
 201                      ['script', '=', 'crunchrunner'],
 202                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 203                      ['docker_image_locator', 'in docker', 'arvados/jobs']])
 204
 205     @mock.patch("arvados.collection.CollectionReader")
 206     def test_done(self, reader):
 207         api = mock.MagicMock()
 208
 209         runner = mock.MagicMock()
 210         runner.api = api
 211         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 212         runner.num_retries = 0
 213         runner.ignore_docker_for_reuse = False
 214
 215         reader().open.return_value = StringIO.StringIO(
 216             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 217 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 218 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 219         """)
 220         api.collections().list().execute.side_effect = ({"items": []},
 221                                                         {"items": [{"manifest_text": "XYZ"}]},
 222                                                         {"items": []},
 223                                                         {"items": [{"manifest_text": "ABC"}]})
 224
 225         arvjob = arvados_cwl.ArvadosJob(runner,
 226                                         mock.MagicMock(),
 227                                         {},
 228                                         None,
 229                                         [],
 230                                         [],
 231                                         "testjob")
 232         arvjob.output_callback = mock.MagicMock()
 233         arvjob.collect_outputs = mock.MagicMock()
 234         arvjob.collect_outputs.return_value = {"out": "stuff"}
 235
 236         arvjob.done({
 237             "state": "Complete",
 238             "output": "99999999999999999999999999999993+99",
 239             "log": "99999999999999999999999999999994+99",
 240             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 241         })
 242
 243         api.collections().list.assert_has_calls([
 244             mock.call(),
 245             # Output collection check
 246             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 247                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 248                           ['name', '=', 'Output 9999999 of testjob']]),
 249             mock.call().execute(num_retries=0),
 250             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 251                  select=['manifest_text']),
 252             mock.call().execute(num_retries=0),
 253             # Log collection's turn
 254             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 255                           ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 256                           ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 257             mock.call().execute(num_retries=0),
 258             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999994+99']],
 259                  select=['manifest_text']),
 260             mock.call().execute(num_retries=0)])
 261
 262         api.collections().create.assert_has_calls([
 263             mock.call(ensure_unique_name=True,
 264                       body={'portable_data_hash': '99999999999999999999999999999993+99',
 265                             'manifest_text': 'XYZ',
 266                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 267                             'name': 'Output 9999999 of testjob'}),
 268             mock.call().execute(num_retries=0),
 269             mock.call(ensure_unique_name=True,
 270                       body={'portable_data_hash': '99999999999999999999999999999994+99',
 271                             'manifest_text': 'ABC',
 272                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 273                             'name': 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 274             mock.call().execute(num_retries=0),
 275         ])
 276
 277         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 278
 279     @mock.patch("arvados.collection.CollectionReader")
 280     def test_done_use_existing_collection(self, reader):
 281         api = mock.MagicMock()
 282
 283         runner = mock.MagicMock()
 284         runner.api = api
 285         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 286         runner.num_retries = 0
 287
 288         reader().open.return_value = StringIO.StringIO(
 289             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 290 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 291 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 292         """)
 293
 294         api.collections().list().execute.side_effect = (
 295             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 296             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 297         )
 298
 299         arvjob = arvados_cwl.ArvadosJob(runner,
 300                                         mock.MagicMock(),
 301                                         {},
 302                                         None,
 303                                         [],
 304                                         [],
 305                                         "testjob")
 306         arvjob.output_callback = mock.MagicMock()
 307         arvjob.collect_outputs = mock.MagicMock()
 308         arvjob.collect_outputs.return_value = {"out": "stuff"}
 309
 310         arvjob.done({
 311             "state": "Complete",
 312             "output": "99999999999999999999999999999993+99",
 313             "log": "99999999999999999999999999999994+99",
 314             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 315         })
 316
 317         api.collections().list.assert_has_calls([
 318             mock.call(),
 319             # Output collection
 320             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 321                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 322                                ['name', '=', 'Output 9999999 of testjob']]),
 323             mock.call().execute(num_retries=0),
 324             # Log collection
 325             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 326                                ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 327                                ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 328             mock.call().execute(num_retries=0)
 329         ])
 330
 331         self.assertFalse(api.collections().create.called)
 332
 333         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 334
 335
 336 class TestWorkflow(unittest.TestCase):
 337     def helper(self, runner, enable_reuse=True):
 338         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 339
 340         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 341                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 342
 343         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 344         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 345         document_loader.fetch_text = document_loader.fetcher.fetch_text
 346         document_loader.check_exists = document_loader.fetcher.check_exists
 347
 348         loadingContext = arvados_cwl.context.ArvLoadingContext(
 349             {"avsc_names": avsc_names,
 350              "basedir": "",
 351              "make_fs_access": make_fs_access,
 352              "loader": document_loader,
 353              "metadata": {"cwlVersion": "v1.0"},
 354              "construct_tool_object": runner.arv_make_tool})
 355         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 356             {"work_api": "jobs",
 357              "basedir": "",
 358              "name": "test_run_wf_"+str(enable_reuse),
 359              "make_fs_access": make_fs_access,
 360              "enable_reuse": enable_reuse,
 361              "priority": 500})
 362
 363         return loadingContext, runtimeContext
 364
 365     # The test passes no builder.resources
 366     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 367     @mock.patch("arvados.collection.CollectionReader")
 368     @mock.patch("arvados.collection.Collection")
 369     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 370     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 371         arvados_cwl.add_arv_hints()
 372
 373         api = mock.MagicMock()
 374         api._rootDesc = get_rootDesc()
 375
 376         runner = arvados_cwl.ArvCwlRunner(api)
 377         self.assertEqual(runner.work_api, 'jobs')
 378
 379         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 380         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 381         runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
 382
 383         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 384         runner.ignore_docker_for_reuse = False
 385         runner.num_retries = 0
 386
 387         loadingContext, runtimeContext = self.helper(runner)
 388
 389         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 390         metadata["cwlVersion"] = tool["cwlVersion"]
 391
 392         mockc = mock.MagicMock()
 393         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 394         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 395
 396         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 397         arvtool.formatgraph = None
 398         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 399
 400         it.next().run(runtimeContext)
 401         it.next().run(runtimeContext)
 402
 403         with open("tests/wf/scatter2_subwf.cwl") as f:
 404             subwf = StripYAMLComments(f.read())
 405
 406         runner.api.jobs().create.assert_called_with(
 407             body=JsonDiffMatcher({
 408                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 409                 'repository': 'arvados',
 410                 'script_version': 'master',
 411                 'script': 'crunchrunner',
 412                 'script_parameters': {
 413                     'tasks': [{'task.env': {
 414                         'HOME': '$(task.outdir)',
 415                         'TMPDIR': '$(task.tmpdir)'},
 416                                'task.vwd': {
 417                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999996+99/workflow.cwl',
 418                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999996+99/cwl.input.yml'
 419                                },
 420                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 421                     'task.stdout': 'cwl.output.json'}]},
 422                 'runtime_constraints': {
 423                     'min_scratch_mb_per_node': 2048,
 424                     'min_cores_per_node': 1,
 425                     'docker_image': 'arvados/jobs',
 426                     'min_ram_mb_per_node': 1024
 427                 },
 428                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 429             filters=[['repository', '=', 'arvados'],
 430                      ['script', '=', 'crunchrunner'],
 431                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 432                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 433             find_or_create=True)
 434
 435         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 436         mockc.open().__enter__().write.assert_has_calls([mock.call(
 437 '''{
 438   "fileblub": {
 439     "basename": "token.txt",
 440     "class": "File",
 441     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 442     "size": 0
 443   },
 444   "sleeptime": 5
 445 }''')])
 446
 447     # The test passes no builder.resources
 448     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 449     @mock.patch("arvados.collection.CollectionReader")
 450     @mock.patch("arvados.collection.Collection")
 451     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 452     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 453         arvados_cwl.add_arv_hints()
 454
 455         api = mock.MagicMock()
 456         api._rootDesc = get_rootDesc()
 457
 458         runner = arvados_cwl.ArvCwlRunner(api)
 459         self.assertEqual(runner.work_api, 'jobs')
 460
 461         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 462         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 463         runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
 464
 465         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 466         runner.ignore_docker_for_reuse = False
 467         runner.num_retries = 0
 468
 469         loadingContext, runtimeContext = self.helper(runner)
 470
 471         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
 472         metadata["cwlVersion"] = tool["cwlVersion"]
 473
 474         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
 475
 476         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 477         arvtool.formatgraph = None
 478         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 479         it.next().run(runtimeContext)
 480         it.next().run(runtimeContext)
 481
 482         with open("tests/wf/echo-subwf.cwl") as f:
 483             subwf = StripYAMLComments(f.read())
 484
 485         runner.api.jobs().create.assert_called_with(
 486             body=JsonDiffMatcher({
 487                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 488                 'repository': 'arvados',
 489                 'script_version': 'master',
 490                 'script': 'crunchrunner',
 491                 'script_parameters': {
 492                     'tasks': [{'task.env': {
 493                         'HOME': '$(task.outdir)',
 494                         'TMPDIR': '$(task.tmpdir)'},
 495                                'task.vwd': {
 496                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999996+99/workflow.cwl',
 497                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999996+99/cwl.input.yml'
 498                                },
 499                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 500                     'task.stdout': 'cwl.output.json'}]},
 501                 'runtime_constraints': {
 502                     'min_scratch_mb_per_node': 4096,
 503                     'min_cores_per_node': 3,
 504                     'docker_image': 'arvados/jobs',
 505                     'min_ram_mb_per_node': 1024
 506                 },
 507                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 508             filters=[['repository', '=', 'arvados'],
 509                      ['script', '=', 'crunchrunner'],
 510                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 511                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 512             find_or_create=True)
 513
 514     def test_default_work_api(self):
 515         arvados_cwl.add_arv_hints()
 516
 517         api = mock.MagicMock()
 518         api._rootDesc = copy.deepcopy(get_rootDesc())
 519         del api._rootDesc.get('resources')['jobs']['methods']['create']
 520         runner = arvados_cwl.ArvCwlRunner(api)
 521         self.assertEqual(runner.work_api, 'containers')