sdk/cwl/tests/test_job.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import str
   8 from builtins import next
   9
  10 import functools
  11 import json
  12 import logging
  13 import mock
  14 import os
  15 import unittest
  16 import copy
  17 import io
  18 import argparse
  19
  20 import arvados
  21 import arvados_cwl
  22 import arvados_cwl.executor
  23 import cwltool.process
  24 from arvados.errors import ApiError
  25 from schema_salad.ref_resolver import Loader
  26 from schema_salad.sourceline import cmap
  27 from .mock_discovery import get_rootDesc
  28 from .matcher import JsonDiffMatcher, StripYAMLComments
  29 from .test_container import CollectionMock
  30 from arvados_cwl.arvdocker import arv_docker_clear_cache
  31
  32 if not os.getenv('ARVADOS_DEBUG'):
  33     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  34     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  35
  36 class TestJob(unittest.TestCase):
  37
  38     def setUp(self):
  39         cwltool.process._names = set()
  40
  41     def helper(self, runner, enable_reuse=True):
  42         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
  43
  44         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  45                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  46         loadingContext = arvados_cwl.context.ArvLoadingContext(
  47             {"avsc_names": avsc_names,
  48              "basedir": "",
  49              "make_fs_access": make_fs_access,
  50              "loader": Loader({}),
  51              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
  52              "makeTool": runner.arv_make_tool})
  53         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  54             {"work_api": "jobs",
  55              "basedir": "",
  56              "name": "test_run_job_"+str(enable_reuse),
  57              "make_fs_access": make_fs_access,
  58              "enable_reuse": enable_reuse,
  59              "priority": 500})
  60
  61         return loadingContext, runtimeContext
  62
  63     # The test passes no builder.resources
  64     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  65     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
  66     def test_run(self, list_images_in_arv):
  67         for enable_reuse in (True, False):
  68             arv_docker_clear_cache()
  69             runner = mock.MagicMock()
  70             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  71             runner.ignore_docker_for_reuse = False
  72             runner.num_retries = 0
  73
  74             list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
  75             runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
  76             # Simulate reused job from another project so that we can check is a can_read
  77             # link is added.
  78             runner.api.jobs().create().execute.return_value = {
  79                 'state': 'Complete' if enable_reuse else 'Queued',
  80                 'owner_uuid': 'zzzzz-tpzed-yyyyyyyyyyyyyyy' if enable_reuse else 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  81                 'uuid': 'zzzzz-819sb-yyyyyyyyyyyyyyy',
  82                 'output': None,
  83             }
  84
  85             tool = cmap({
  86                 "inputs": [],
  87                 "outputs": [],
  88                 "baseCommand": "ls",
  89                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  90                 "id": "#",
  91                 "class": "CommandLineTool"
  92             })
  93
  94             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
  95
  96             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
  97             arvtool.formatgraph = None
  98             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
  99                 j.run(runtimeContext)
 100                 runner.api.jobs().create.assert_called_with(
 101                     body=JsonDiffMatcher({
 102                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 103                         'runtime_constraints': {},
 104                         'script_parameters': {
 105                             'tasks': [{
 106                                 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
 107                                 'command': ['ls', '$(task.outdir)']
 108                             }],
 109                         },
 110                         'script_version': 'master',
 111                         'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 112                         'repository': 'arvados',
 113                         'script': 'crunchrunner',
 114                         'runtime_constraints': {
 115                             'docker_image': 'arvados/jobs',
 116                             'min_cores_per_node': 1,
 117                             'min_ram_mb_per_node': 1024,
 118                             'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
 119                         }
 120                     }),
 121                     find_or_create=enable_reuse,
 122                     filters=[['repository', '=', 'arvados'],
 123                              ['script', '=', 'crunchrunner'],
 124                              ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 125                              ['docker_image_locator', 'in docker', 'arvados/jobs']]
 126                 )
 127                 if enable_reuse:
 128                     runner.api.links().create.assert_called_with(
 129                         body=JsonDiffMatcher({
 130                             'link_class': 'permission',
 131                             'name': 'can_read',
 132                             "tail_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 133                             "head_uuid": "zzzzz-819sb-yyyyyyyyyyyyyyy",
 134                         })
 135                     )
 136                     # Simulate an API excepction when trying to create a
 137                     # sharing link on the job
 138                     runner.api.links().create.side_effect = ApiError(
 139                         mock.MagicMock(return_value={'status': 403}),
 140                         bytes(b'Permission denied'))
 141                     j.run(runtimeContext)
 142                 else:
 143                     assert not runner.api.links().create.called
 144
 145     # The test passes some fields in builder.resources
 146     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 147     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 148     def test_resource_requirements(self, list_images_in_arv):
 149         runner = mock.MagicMock()
 150         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 151         runner.ignore_docker_for_reuse = False
 152         runner.num_retries = 0
 153         arvados_cwl.add_arv_hints()
 154
 155         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 156         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 157
 158         tool = {
 159             "inputs": [],
 160             "outputs": [],
 161             "hints": [{
 162                 "class": "ResourceRequirement",
 163                 "coresMin": 3,
 164                 "ramMin": 3000,
 165                 "tmpdirMin": 4000
 166             }, {
 167                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 168                 "keep_cache": 512,
 169                 "outputDirType": "keep_output_dir"
 170             }, {
 171                 "class": "http://arvados.org/cwl#APIRequirement",
 172             },
 173             {
 174                 "class": "http://arvados.org/cwl#ReuseRequirement",
 175                 "enableReuse": False
 176             }],
 177             "baseCommand": "ls",
 178             "id": "#",
 179             "class": "CommandLineTool"
 180         }
 181
 182         loadingContext, runtimeContext = self.helper(runner)
 183
 184         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 185         arvtool.formatgraph = None
 186         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 187             j.run(runtimeContext)
 188         runner.api.jobs().create.assert_called_with(
 189             body=JsonDiffMatcher({
 190                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 191                 'runtime_constraints': {},
 192                 'script_parameters': {
 193                     'tasks': [{
 194                         'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
 195                         'task.keepTmpOutput': True,
 196                         'command': ['ls']
 197                     }]
 198             },
 199             'script_version': 'master',
 200                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 201                 'repository': 'arvados',
 202                 'script': 'crunchrunner',
 203                 'runtime_constraints': {
 204                     'docker_image': 'arvados/jobs',
 205                     'min_cores_per_node': 3,
 206                     'min_ram_mb_per_node': 3512,     # ramMin + keep_cache
 207                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
 208                     'keep_cache_mb_per_task': 512
 209                 }
 210             }),
 211             find_or_create=False,
 212             filters=[['repository', '=', 'arvados'],
 213                      ['script', '=', 'crunchrunner'],
 214                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 215                      ['docker_image_locator', 'in docker', 'arvados/jobs']])
 216
 217     @mock.patch("arvados.collection.CollectionReader")
 218     def test_done(self, reader):
 219         api = mock.MagicMock()
 220
 221         runner = mock.MagicMock()
 222         runner.api = api
 223         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 224         runner.num_retries = 0
 225         runner.ignore_docker_for_reuse = False
 226
 227         reader().keys.return_value = "log.txt"
 228         reader().open.return_value = io.StringIO(
 229             str(u"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 230 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 231 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 232         """))
 233         api.collections().list().execute.side_effect = ({"items": []},
 234                                                         {"items": [{"manifest_text": "XYZ"}]},
 235                                                         {"items": []},
 236                                                         {"items": [{"manifest_text": "ABC"}]})
 237
 238         arvjob = arvados_cwl.ArvadosJob(runner,
 239                                         mock.MagicMock(),
 240                                         {},
 241                                         None,
 242                                         [],
 243                                         [],
 244                                         "testjob")
 245         arvjob.output_callback = mock.MagicMock()
 246         arvjob.collect_outputs = mock.MagicMock()
 247         arvjob.collect_outputs.return_value = {"out": "stuff"}
 248
 249         arvjob.done({
 250             "state": "Complete",
 251             "output": "99999999999999999999999999999993+99",
 252             "log": "99999999999999999999999999999994+99",
 253             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 254         })
 255
 256         api.collections().list.assert_has_calls([
 257             mock.call(),
 258             # Output collection check
 259             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 260                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 261                           ['name', '=', 'Output 9999999 of testjob']]),
 262             mock.call().execute(num_retries=0),
 263             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 264                  select=['manifest_text']),
 265             mock.call().execute(num_retries=0),
 266             # Log collection's turn
 267             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 268                           ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 269                           ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 270             mock.call().execute(num_retries=0),
 271             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999994+99']],
 272                  select=['manifest_text']),
 273             mock.call().execute(num_retries=0)])
 274
 275         api.collections().create.assert_has_calls([
 276             mock.call(ensure_unique_name=True,
 277                       body={'portable_data_hash': '99999999999999999999999999999993+99',
 278                             'manifest_text': 'XYZ',
 279                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 280                             'name': 'Output 9999999 of testjob'}),
 281             mock.call().execute(num_retries=0),
 282             mock.call(ensure_unique_name=True,
 283                       body={'portable_data_hash': '99999999999999999999999999999994+99',
 284                             'manifest_text': 'ABC',
 285                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 286                             'name': 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 287             mock.call().execute(num_retries=0),
 288         ])
 289
 290         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 291
 292     @mock.patch("arvados.collection.CollectionReader")
 293     def test_done_use_existing_collection(self, reader):
 294         api = mock.MagicMock()
 295
 296         runner = mock.MagicMock()
 297         runner.api = api
 298         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 299         runner.num_retries = 0
 300
 301         reader().keys.return_value = "log.txt"
 302         reader().open.return_value = io.StringIO(
 303             str(u"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 304 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 305 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
 306         """))
 307
 308         api.collections().list().execute.side_effect = (
 309             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 310             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
 311         )
 312
 313         arvjob = arvados_cwl.ArvadosJob(runner,
 314                                         mock.MagicMock(),
 315                                         {},
 316                                         None,
 317                                         [],
 318                                         [],
 319                                         "testjob")
 320         arvjob.output_callback = mock.MagicMock()
 321         arvjob.collect_outputs = mock.MagicMock()
 322         arvjob.collect_outputs.return_value = {"out": "stuff"}
 323
 324         arvjob.done({
 325             "state": "Complete",
 326             "output": "99999999999999999999999999999993+99",
 327             "log": "99999999999999999999999999999994+99",
 328             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 329         })
 330
 331         api.collections().list.assert_has_calls([
 332             mock.call(),
 333             # Output collection
 334             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 335                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 336                                ['name', '=', 'Output 9999999 of testjob']]),
 337             mock.call().execute(num_retries=0),
 338             # Log collection
 339             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 340                                ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
 341                                ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
 342             mock.call().execute(num_retries=0)
 343         ])
 344
 345         self.assertFalse(api.collections().create.called)
 346
 347         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 348
 349
 350 class TestWorkflow(unittest.TestCase):
 351
 352     def setUp(self):
 353         cwltool.process._names = set()
 354
 355     def helper(self, runner, enable_reuse=True):
 356         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 357
 358         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 359                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 360
 361         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 362         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 363         document_loader.fetch_text = document_loader.fetcher.fetch_text
 364         document_loader.check_exists = document_loader.fetcher.check_exists
 365
 366         loadingContext = arvados_cwl.context.ArvLoadingContext(
 367             {"avsc_names": avsc_names,
 368              "basedir": "",
 369              "make_fs_access": make_fs_access,
 370              "loader": document_loader,
 371              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 372              "construct_tool_object": runner.arv_make_tool})
 373         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 374             {"work_api": "jobs",
 375              "basedir": "",
 376              "name": "test_run_wf_"+str(enable_reuse),
 377              "make_fs_access": make_fs_access,
 378              "enable_reuse": enable_reuse,
 379              "priority": 500})
 380
 381         return loadingContext, runtimeContext
 382
 383     # The test passes no builder.resources
 384     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 385     @mock.patch("arvados.collection.CollectionReader")
 386     @mock.patch("arvados.collection.Collection")
 387     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 388     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 389         arv_docker_clear_cache()
 390         arvados_cwl.add_arv_hints()
 391
 392         api = mock.MagicMock()
 393         api._rootDesc = get_rootDesc()
 394
 395         runner = arvados_cwl.executor.ArvCwlExecutor(api, argparse.Namespace(work_api="jobs",
 396                                                                              output_name=None,
 397                                                                              output_tags=None,
 398                                                                              thread_count=1,
 399                                                                              collection_cache_size=None))
 400         self.assertEqual(runner.work_api, 'jobs')
 401
 402         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 403         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 404         runner.api.collections().list().execute.return_value = {"items": [{
 405             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 406             "portable_data_hash": "99999999999999999999999999999993+99"}]}
 407
 408         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 409         runner.ignore_docker_for_reuse = False
 410         runner.num_retries = 0
 411
 412         loadingContext, runtimeContext = self.helper(runner)
 413         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 414         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 415         metadata["cwlVersion"] = tool["cwlVersion"]
 416
 417         mockc = mock.MagicMock()
 418         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 419         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 420
 421         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 422         arvtool.formatgraph = None
 423         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 424
 425         next(it).run(runtimeContext)
 426         next(it).run(runtimeContext)
 427
 428         with open("tests/wf/scatter2_subwf.cwl") as f:
 429             subwf = StripYAMLComments(f.read().rstrip())
 430
 431         runner.api.jobs().create.assert_called_with(
 432             body=JsonDiffMatcher({
 433                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 434                 'repository': 'arvados',
 435                 'script_version': 'master',
 436                 'script': 'crunchrunner',
 437                 'script_parameters': {
 438                     'tasks': [{'task.env': {
 439                         'HOME': '$(task.outdir)',
 440                         'TMPDIR': '$(task.tmpdir)'},
 441                                'task.vwd': {
 442                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999996+99/workflow.cwl',
 443                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999996+99/cwl.input.yml'
 444                                },
 445                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 446                     'task.stdout': 'cwl.output.json'}]},
 447                 'runtime_constraints': {
 448                     'min_scratch_mb_per_node': 2048,
 449                     'min_cores_per_node': 1,
 450                     'docker_image': 'arvados/jobs',
 451                     'min_ram_mb_per_node': 1024
 452                 },
 453                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 454             filters=[['repository', '=', 'arvados'],
 455                      ['script', '=', 'crunchrunner'],
 456                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 457                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 458             find_or_create=True)
 459
 460         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 461         mockc.open().__enter__().write.assert_has_calls([mock.call(
 462 bytes(b'''{
 463   "fileblub": {
 464     "basename": "token.txt",
 465     "class": "File",
 466     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 467     "size": 0
 468   },
 469   "sleeptime": 5
 470 }'''))])
 471
 472     # The test passes no builder.resources
 473     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 474     @mock.patch("arvados.collection.CollectionReader")
 475     @mock.patch("arvados.collection.Collection")
 476     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 477     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 478         arv_docker_clear_cache()
 479         arvados_cwl.add_arv_hints()
 480
 481         api = mock.MagicMock()
 482         api._rootDesc = get_rootDesc()
 483
 484         runner = arvados_cwl.executor.ArvCwlExecutor(api, argparse.Namespace(work_api="jobs",
 485                                                                              output_name=None,
 486                                                                              output_tags=None,
 487                                                                              thread_count=1,
 488                                                                              collection_cache_size=None))
 489         self.assertEqual(runner.work_api, 'jobs')
 490
 491         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 492         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 493         runner.api.collections().list().execute.return_value = {"items": [{
 494             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 495             "portable_data_hash": "99999999999999999999999999999993+99"}]}
 496
 497         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 498         runner.ignore_docker_for_reuse = False
 499         runner.num_retries = 0
 500
 501         loadingContext, runtimeContext = self.helper(runner)
 502         loadingContext.do_update = True
 503         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 504         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
 505
 506         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
 507
 508         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 509         arvtool.formatgraph = None
 510         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 511
 512         next(it).run(runtimeContext)
 513         next(it).run(runtimeContext)
 514
 515         with open("tests/wf/echo-subwf.cwl") as f:
 516             subwf = StripYAMLComments(f.read())
 517
 518         runner.api.jobs().create.assert_called_with(
 519             body=JsonDiffMatcher({
 520                 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d',
 521                 'repository': 'arvados',
 522                 'script_version': 'master',
 523                 'script': 'crunchrunner',
 524                 'script_parameters': {
 525                     'tasks': [{'task.env': {
 526                         'HOME': '$(task.outdir)',
 527                         'TMPDIR': '$(task.tmpdir)'},
 528                                'task.vwd': {
 529                                    'workflow.cwl': '$(task.keep)/99999999999999999999999999999996+99/workflow.cwl',
 530                                    'cwl.input.yml': '$(task.keep)/99999999999999999999999999999996+99/cwl.input.yml'
 531                                },
 532                     'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
 533                     'task.stdout': 'cwl.output.json'}]},
 534                 'runtime_constraints': {
 535                     'min_scratch_mb_per_node': 4096,
 536                     'min_cores_per_node': 3,
 537                     'docker_image': 'arvados/jobs',
 538                     'min_ram_mb_per_node': 1024
 539                 },
 540                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
 541             filters=[['repository', '=', 'arvados'],
 542                      ['script', '=', 'crunchrunner'],
 543                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
 544                      ['docker_image_locator', 'in docker', 'arvados/jobs']],
 545             find_or_create=True)
 546
 547     def test_default_work_api(self):
 548         arvados_cwl.add_arv_hints()
 549
 550         api = mock.MagicMock()
 551         api._rootDesc = copy.deepcopy(get_rootDesc())
 552         del api._rootDesc.get('resources')['jobs']['methods']['create']
 553         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 554         self.assertEqual(runner.work_api, 'containers')