sdk/cwl/tests/test_job.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import functools
   6 import json
   7 import logging
   8 import mock
   9 import os
  10 import unittest
  11 import copy
  12 import StringIO
  13
  14 import arvados
  15 import arvados_cwl
  16 import cwltool.process
  17 from arvados.errors import ApiError
  18 from schema_salad.ref_resolver import Loader
  19 from schema_salad.sourceline import cmap
  20 from .mock_discovery import get_rootDesc
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22
  23 if not os.getenv('ARVADOS_DEBUG'):
  24     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  25     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  26
  27 class TestJob(unittest.TestCase):
  28
  29     def helper(self, runner, enable_reuse=True):
  30         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  31
  32         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  33                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  34         loadingContext = arvados_cwl.context.ArvLoadingContext(
  35             {"avsc_names": avsc_names,
  36              "basedir": "",
  37              "make_fs_access": make_fs_access,
  38              "loader": Loader({}),
  39              "metadata": {"cwlVersion": "v1.0"},
  40              "makeTool": runner.arv_make_tool})
  41         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  42             {"work_api": "jobs",
  43              "basedir": "",
  44              "name": "test_run_job_"+str(enable_reuse),
  45              "make_fs_access": make_fs_access,
  46              "enable_reuse": enable_reuse,
  47              "priority": 500})
  48
  49         return loadingContext, runtimeContext
  50
  51     # The test passes no builder.resources
  52     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  53     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
  54     def test_run(self, list_images_in_arv):
  55         for enable_reuse in (True, False):
  56             runner = mock.MagicMock()
  57             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  58             runner.ignore_docker_for_reuse = False
  59             runner.num_retries = 0
  60
  61             list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
  62             runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
  63             # Simulate reused job from another project so that we can check is a can_read
  64             # link is added.
  65             runner.api.jobs().create().execute.return_value = {
  66                 'state': 'Complete' if enable_reuse else 'Queued',
  67                 'owner_uuid': 'zzzzz-tpzed-yyyyyyyyyyyyyyy' if enable_reuse else 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  68                 'uuid': 'zzzzz-819sb-yyyyyyyyyyyyyyy',
  69                 'output': None,
  70             }
  71
  72             tool = cmap({
  73                 "inputs": [],
  74                 "outputs": [],
  75                 "baseCommand": "ls",
  76                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  77                 "id": "#",
  78                 "class": "CommandLineTool"
  79             })
  80
  81             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
  82
  83             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
  84             arvtool.formatgraph = None
  85             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
  86                 j.run(runtimeContext)
  87                 runner.api.jobs().create.assert_called_with(
  88                     body=JsonDiffMatcher({
  89                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  90                         'runtime_constraints': {},
  91                         'script_parameters': {
  92                             'tasks': [{
  93                                 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
  94                                 'command': ['ls', '$(task.outdir)']
  95                             }],
  96                         },
  97                         'script_version': 'master',
  98                         'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
  99                         'repository': 'arvados',
 100                         'script': 'crunchrunner',
 101                         'runtime_constraints': {
 102                             'docker_image': 'arvados/jobs',
 103                             'min_cores_per_node': 1,
 104                             'min_ram_mb_per_node': 1024,
 105                             'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
 106                         }
 107                     }),
 108                     find_or_create=enable_reuse,
 109                     filters=[['repository', '=', 'arvados'],
 110                              ['script', '=', 'crunchrunner'],
 111                              ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 112                              ['docker_image_locator', 'in docker', 'arvados/jobs']]
 113                 )
 114                 if enable_reuse:
 115                     runner.api.links().create.assert_called_with(
 116                         body=JsonDiffMatcher({
 117                             'link_class': 'permission',
 118                             'name': 'can_read',
 119                             "tail_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 120                             "head_uuid": "zzzzz-819sb-yyyyyyyyyyyyyyy",
 121                         })
 122                     )
 123                     # Simulate an API excepction when trying to create a
 124                     # sharing link on the job
 125                     runner.api.links().create.side_effect = ApiError(
 126                         mock.MagicMock(return_value={'status': 403}),
 127                         'Permission denied')
 128                     j.run(runtimeContext)
 129                 else:
 130                     assert not runner.api.links().create.called
 131
 132     # The test passes some fields in builder.resources
 133     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 134     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 135     def test_resource_requirements(self, list_images_in_arv):
 136         runner = mock.MagicMock()
 137         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 138         runner.ignore_docker_for_reuse = False
 139         runner.num_retries = 0
 140         arvados_cwl.add_arv_hints()
 141
 142         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 143         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 144
 145         tool = {
 146             "inputs": [],
 147             "outputs": [],
 148             "hints": [{
 149                 "class": "ResourceRequirement",
 150                 "coresMin": 3,
 151                 "ramMin": 3000,
 152                 "tmpdirMin": 4000
 153             }, {
 154                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 155                 "keep_cache": 512,
 156                 "outputDirType": "keep_output_dir"
 157             }, {
 158                 "class": "http://arvados.org/cwl#APIRequirement",
 159             },
 160             {
 161                 "class": "http://arvados.org/cwl#ReuseRequirement",
 162                 "enableReuse": False
 163             }],
 164             "baseCommand": "ls",
 165             "id": "#",
 166             "class": "CommandLineTool"
 167         }
 168
 169         loadingContext, runtimeContext = self.helper(runner)
 170
 171         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 172         arvtool.formatgraph = None
 173         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 174             j.run(runtimeContext)
 175         runner.api.jobs().create.assert_called_with(
 176             body=JsonDiffMatcher({
 177                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 178                 'runtime_constraints': {},
 179                 'script_parameters': {
 180                     'tasks': [{
 181                         'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
 182                         'task.keepTmpOutput': True,
 183                         'command': ['ls']
 184                     }]
 185             },
 186             'script_version': 'master',
 187                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 188                 'repository': 'arvados',
 189                 'script': 'crunchrunner',
 190                 'runtime_constraints': {
 191                     'docker_image': 'arvados/jobs',
 192                     'min_cores_per_node': 3,
 193                     'min_ram_mb_per_node': 3512,     # ramMin + keep_cache
 194                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
 195                     'keep_cache_mb_per_task': 512
 196                 }
 197             }),
 198             find_or_create=False,
 199             filters=[['repository', '=', 'arvados'],
 200                      ['script', '=', 'crunchrunner'],
 201                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 202                      ['docker_image_locator', 'in docker', 'arvados/jobs']])
 203
 204     @mock.patch("arvados.collection.CollectionReader")
 205     def test_done(self, reader):
 206         api = mock.MagicMock()
 207
 208         runner = mock.MagicMock()
 209         runner.api = api
 210         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 211         runner.num_retries = 0
 212         runner.ignore_docker_for_reuse = False
 213
 214         reader().open.return_value = StringIO.StringIO(
 215             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 216 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 217 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 218         """)
 219         api.collections().list().execute.side_effect = ({"items": []},
 220                                                         {"items": [{"manifest_text": "XYZ"}]},
 221                                                         {"items": []},
 222                                                         {"items": [{"manifest_text": "ABC"}]})
 223
 224         arvjob = arvados_cwl.ArvadosJob(runner,
 225                                         mock.MagicMock(),
 226                                         {},
 227                                         None,
 228                                         [],
 229                                         [],
 230                                         "testjob")
 231         arvjob.output_callback = mock.MagicMock()
 232         arvjob.collect_outputs = mock.MagicMock()
 233         arvjob.collect_outputs.return_value = {"out": "stuff"}
 234
 235         arvjob.done({
 236             "state": "Complete",
 237             "output": "99999999999999999999999999999993+99",
 238             "log": "99999999999999999999999999999994+99",
 239             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 240         })
 241
 242         api.collections().list.assert_has_calls([
 243             mock.call(),
 244             # Output collection check
 245             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 246                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 247                           ['name', '=', 'Output 9999999 of testjob']]),
 248             mock.call().execute(num_retries=0),
 249             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 250                  select=['manifest_text']),
 251             mock.call().execute(num_retries=0),
 252             # Log collection's turn
 253             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 254                           ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 255                           ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 256             mock.call().execute(num_retries=0),
 257             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999994+99']],
 258                  select=['manifest_text']),
 259             mock.call().execute(num_retries=0)])
 260
 261         api.collections().create.assert_has_calls([
 262             mock.call(ensure_unique_name=True,
 263                       body={'portable_data_hash': '99999999999999999999999999999993+99',
 264                             'manifest_text': 'XYZ',
 265                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 266                             'name': 'Output 9999999 of testjob'}),
 267             mock.call().execute(num_retries=0),
 268             mock.call(ensure_unique_name=True,
 269                       body={'portable_data_hash': '99999999999999999999999999999994+99',
 270                             'manifest_text': 'ABC',
 271                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272                             'name': 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 273             mock.call().execute(num_retries=0),
 274         ])
 275
 276         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 277
 278     @mock.patch("arvados.collection.CollectionReader")
 279     def test_done_use_existing_collection(self, reader):
 280         api = mock.MagicMock()
 281
 282         runner = mock.MagicMock()
 283         runner.api = api
 284         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 285         runner.num_retries = 0
 286
 287         reader().open.return_value = StringIO.StringIO(
 288             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 289 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 290 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 291         """)
 292
 293         api.collections().list().execute.side_effect = (
 294             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 295             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 296         )
 297
 298         arvjob = arvados_cwl.ArvadosJob(runner,
 299                                         mock.MagicMock(),
 300                                         {},
 301                                         None,
 302                                         [],
 303                                         [],
 304                                         "testjob")
 305         arvjob.output_callback = mock.MagicMock()
 306         arvjob.collect_outputs = mock.MagicMock()
 307         arvjob.collect_outputs.return_value = {"out": "stuff"}
 308
 309         arvjob.done({
 310             "state": "Complete",
 311             "output": "99999999999999999999999999999993+99",
 312             "log": "99999999999999999999999999999994+99",
 313             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 314         })
 315
 316         api.collections().list.assert_has_calls([
 317             mock.call(),
 318             # Output collection
 319             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 320                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 321                                ['name', '=', 'Output 9999999 of testjob']]),
 322             mock.call().execute(num_retries=0),
 323             # Log collection
 324             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 325                                ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 326                                ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 327             mock.call().execute(num_retries=0)
 328         ])
 329
 330         self.assertFalse(api.collections().create.called)
 331
 332         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 333
 334
 335 class TestWorkflow(unittest.TestCase):
 336     def helper(self, runner, enable_reuse=True):
 337         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 338
 339         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 340                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 341
 342         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 343         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 344         document_loader.fetch_text = document_loader.fetcher.fetch_text
 345         document_loader.check_exists = document_loader.fetcher.check_exists
 346
 347         loadingContext = arvados_cwl.context.ArvLoadingContext(
 348             {"avsc_names": avsc_names,
 349              "basedir": "",
 350              "make_fs_access": make_fs_access,
 351              "loader": document_loader,
 352              "metadata": {"cwlVersion": "v1.0"},
 353              "construct_tool_object": runner.arv_make_tool})
 354         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 355             {"work_api": "jobs",
 356              "basedir": "",
 357              "name": "test_run_wf_"+str(enable_reuse),
 358              "make_fs_access": make_fs_access,
 359              "enable_reuse": enable_reuse,
 360              "priority": 500})
 361
 362         return loadingContext, runtimeContext
 363
 364     # The test passes no builder.resources
 365     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 366     @mock.patch("arvados.collection.CollectionReader")
 367     @mock.patch("arvados.collection.Collection")
 368     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 369     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 370         arvados_cwl.add_arv_hints()
 371
 372         api = mock.MagicMock()
 373         api._rootDesc = get_rootDesc()
 374
 375         runner = arvados_cwl.ArvCwlRunner(api)
 376         self.assertEqual(runner.work_api, 'jobs')
 377
 378         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 379         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 380         runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
 381
 382         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 383         runner.ignore_docker_for_reuse = False
 384         runner.num_retries = 0
 385
 386         loadingContext, runtimeContext = self.helper(runner)
 387
 388         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 389         metadata["cwlVersion"] = tool["cwlVersion"]
 390
 391         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
 392
 393         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 394         arvtool.formatgraph = None
 395         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 396
 397         it.next().run(runtimeContext)
 398         it.next().run(runtimeContext)
 399
 400         with open("tests/wf/scatter2_subwf.cwl") as f:
 401             subwf = StripYAMLComments(f.read())
 402
 403         runner.api.jobs().create.assert_called_with(
 404             body=JsonDiffMatcher({
 405                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 406                 'repository': 'arvados',
 407                 'script_version': 'master',
 408                 'script': 'crunchrunner',
 409                 'script_parameters': {
 410                     'tasks': [{'task.env': {
 411                         'HOME': '$(task.outdir)',
 412                         'TMPDIR': '$(task.tmpdir)'},
 413                                'task.vwd': {
 414                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
 415                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
 416                                },
 417                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 418                     'task.stdout': 'cwl.output.json'}]},
 419                 'runtime_constraints': {
 420                     'min_scratch_mb_per_node': 2048,
 421                     'min_cores_per_node': 1,
 422                     'docker_image': 'arvados/jobs',
 423                     'min_ram_mb_per_node': 1024
 424                 },
 425                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 426             filters=[['repository', '=', 'arvados'],
 427                      ['script', '=', 'crunchrunner'],
 428                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 429                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 430             find_or_create=True)
 431
 432         mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
 433         mockcollection().open().__enter__().write.assert_has_calls([mock.call(
 434 '''{
 435   "fileblub": {
 436     "basename": "token.txt",
 437     "class": "File",
 438     "location": "/keep/99999999999999999999999999999999+118/token.txt"
 439   },
 440   "sleeptime": 5
 441 }''')])
 442
 443     # The test passes no builder.resources
 444     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 445     @mock.patch("arvados.collection.CollectionReader")
 446     @mock.patch("arvados.collection.Collection")
 447     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 448     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 449         arvados_cwl.add_arv_hints()
 450
 451         api = mock.MagicMock()
 452         api._rootDesc = get_rootDesc()
 453
 454         runner = arvados_cwl.ArvCwlRunner(api)
 455         self.assertEqual(runner.work_api, 'jobs')
 456
 457         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 458         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
 459         runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
 460
 461         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 462         runner.ignore_docker_for_reuse = False
 463         runner.num_retries = 0
 464
 465         loadingContext, runtimeContext = self.helper(runner)
 466
 467         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
 468         metadata["cwlVersion"] = tool["cwlVersion"]
 469
 470         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
 471
 472         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 473         arvtool.formatgraph = None
 474         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 475         it.next().run(runtimeContext)
 476         it.next().run(runtimeContext)
 477
 478         with open("tests/wf/echo-subwf.cwl") as f:
 479             subwf = StripYAMLComments(f.read())
 480
 481         runner.api.jobs().create.assert_called_with(
 482             body=JsonDiffMatcher({
 483                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 484                 'repository': 'arvados',
 485                 'script_version': 'master',
 486                 'script': 'crunchrunner',
 487                 'script_parameters': {
 488                     'tasks': [{'task.env': {
 489                         'HOME': '$(task.outdir)',
 490                         'TMPDIR': '$(task.tmpdir)'},
 491                                'task.vwd': {
 492                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
 493                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
 494                                },
 495                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 496                     'task.stdout': 'cwl.output.json'}]},
 497                 'runtime_constraints': {
 498                     'min_scratch_mb_per_node': 4096,
 499                     'min_cores_per_node': 3,
 500                     'docker_image': 'arvados/jobs',
 501                     'min_ram_mb_per_node': 1024
 502                 },
 503                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 504             filters=[['repository', '=', 'arvados'],
 505                      ['script', '=', 'crunchrunner'],
 506                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 507                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 508             find_or_create=True)
 509
 510     def test_default_work_api(self):
 511         arvados_cwl.add_arv_hints()
 512
 513         api = mock.MagicMock()
 514         api._rootDesc = copy.deepcopy(get_rootDesc())
 515         del api._rootDesc.get('resources')['jobs']['methods']['create']
 516         runner = arvados_cwl.ArvCwlRunner(api)
 517         self.assertEqual(runner.work_api, 'containers')