sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from cwltool.update import INTERNAL_VERSION
  22 from schema_salad.ref_resolver import Loader
  23 from schema_salad.sourceline import cmap
  24
  25 from .matcher import JsonDiffMatcher, StripYAMLComments
  26 from .mock_discovery import get_rootDesc
  27
  28 if not os.getenv('ARVADOS_DEBUG'):
  29     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  30     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  31
  32 class CollectionMock(object):
  33     def __init__(self, vwdmock, *args, **kwargs):
  34         self.vwdmock = vwdmock
  35         self.count = 0
  36
  37     def open(self, *args, **kwargs):
  38         self.count += 1
  39         return self.vwdmock.open(*args, **kwargs)
  40
  41     def copy(self, *args, **kwargs):
  42         self.count += 1
  43         self.vwdmock.copy(*args, **kwargs)
  44
  45     def save_new(self, *args, **kwargs):
  46         pass
  47
  48     def __len__(self):
  49         return self.count
  50
  51     def portable_data_hash(self):
  52         if self.count == 0:
  53             return arvados.config.EMPTY_BLOCK_LOCATOR
  54         else:
  55             return "99999999999999999999999999999996+99"
  56
  57
  58 class TestContainer(unittest.TestCase):
  59
  60     def setUp(self):
  61         cwltool.process._names = set()
  62         arv_docker_clear_cache()
  63
  64     def helper(self, runner, enable_reuse=True):
  65         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  66
  67         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  68                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  69         loadingContext = arvados_cwl.context.ArvLoadingContext(
  70             {"avsc_names": avsc_names,
  71              "basedir": "",
  72              "make_fs_access": make_fs_access,
  73              "loader": Loader({}),
  74              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  75         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  76             {"work_api": "containers",
  77              "basedir": "",
  78              "name": "test_run_"+str(enable_reuse),
  79              "make_fs_access": make_fs_access,
  80              "tmpdir": "/tmp",
  81              "enable_reuse": enable_reuse,
  82              "priority": 500,
  83              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84             })
  85
  86         return loadingContext, runtimeContext
  87
  88     # Helper function to set up the ArvCwlExecutor to use the containers api
  89     # and test that the RuntimeStatusLoggingHandler is set up correctly
  90     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  91         api = mock.MagicMock()
  92         api._rootDesc = copy.deepcopy(get_rootDesc())
  93
  94         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  95         # adds the logging handler that will call runtime_status_update() mock
  96         self.assertFalse(gcc_mock.called)
  97         runner = arvados_cwl.ArvCwlExecutor(api)
  98         self.assertEqual(runner.work_api, 'containers')
  99         root_logger = logging.getLogger('')
 100         handlerClasses = [h.__class__ for h in root_logger.handlers]
 101         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 102         return runner
 103
 104     # The test passes no builder.resources
 105     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 106     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 107     def test_run(self, keepdocker):
 108         for enable_reuse in (True, False):
 109             arv_docker_clear_cache()
 110
 111             runner = mock.MagicMock()
 112             runner.ignore_docker_for_reuse = False
 113             runner.intermediate_output_ttl = 0
 114             runner.secret_store = cwltool.secrets.SecretStore()
 115
 116             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 117             runner.api.collections().get().execute.return_value = {
 118                 "portable_data_hash": "99999999999999999999999999999993+99"}
 119
 120             tool = cmap({
 121                 "inputs": [],
 122                 "outputs": [],
 123                 "baseCommand": "ls",
 124                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 125                 "id": "#",
 126                 "class": "org.w3id.cwl.cwl.CommandLineTool"
 127             })
 128
 129             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 130
 131             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 132             arvtool.formatgraph = None
 133
 134             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 135                 j.run(runtimeContext)
 136                 runner.api.container_requests().create.assert_called_with(
 137                     body=JsonDiffMatcher({
 138                         'environment': {
 139                             'HOME': '/var/spool/cwl',
 140                             'TMPDIR': '/tmp'
 141                         },
 142                         'name': 'test_run_'+str(enable_reuse),
 143                         'runtime_constraints': {
 144                             'vcpus': 1,
 145                             'ram': 1073741824
 146                         },
 147                         'use_existing': enable_reuse,
 148                         'priority': 500,
 149                         'mounts': {
 150                             '/tmp': {'kind': 'tmp',
 151                                      "capacity": 1073741824
 152                                  },
 153                             '/var/spool/cwl': {'kind': 'tmp',
 154                                                "capacity": 1073741824 }
 155                         },
 156                         'state': 'Committed',
 157                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 158                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 159                         'output_path': '/var/spool/cwl',
 160                         'output_ttl': 0,
 161                         'container_image': '99999999999999999999999999999993+99',
 162                         'command': ['ls', '/var/spool/cwl'],
 163                         'cwd': '/var/spool/cwl',
 164                         'scheduling_parameters': {},
 165                         'properties': {},
 166                         'secret_mounts': {},
 167                         'output_storage_classes': ["default"]
 168                     }))
 169
 170     # The test passes some fields in builder.resources
 171     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 172     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 173     def test_resource_requirements(self, keepdocker):
 174         runner = mock.MagicMock()
 175         runner.ignore_docker_for_reuse = False
 176         runner.intermediate_output_ttl = 3600
 177         runner.secret_store = cwltool.secrets.SecretStore()
 178
 179         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 180         runner.api.collections().get().execute.return_value = {
 181             "portable_data_hash": "99999999999999999999999999999993+99"}
 182
 183         tool = cmap({
 184             "inputs": [],
 185             "outputs": [],
 186             "hints": [{
 187                 "class": "ResourceRequirement",
 188                 "coresMin": 3,
 189                 "ramMin": 3000,
 190                 "tmpdirMin": 4000,
 191                 "outdirMin": 5000
 192             }, {
 193                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 194                 "keep_cache": 512
 195             }, {
 196                 "class": "http://arvados.org/cwl#APIRequirement",
 197             }, {
 198                 "class": "http://arvados.org/cwl#PartitionRequirement",
 199                 "partition": "blurb"
 200             }, {
 201                 "class": "http://arvados.org/cwl#IntermediateOutput",
 202                 "outputTTL": 7200
 203             }, {
 204                 "class": "http://arvados.org/cwl#ReuseRequirement",
 205                 "enableReuse": False
 206             }],
 207             "baseCommand": "ls",
 208             "id": "#",
 209             "class": "org.w3id.cwl.cwl.CommandLineTool"
 210         })
 211
 212         loadingContext, runtimeContext = self.helper(runner)
 213         runtimeContext.name = "test_resource_requirements"
 214
 215         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 216         arvtool.formatgraph = None
 217         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 218             j.run(runtimeContext)
 219
 220         call_args, call_kwargs = runner.api.container_requests().create.call_args
 221
 222         call_body_expected = {
 223             'environment': {
 224                 'HOME': '/var/spool/cwl',
 225                 'TMPDIR': '/tmp'
 226             },
 227             'name': 'test_resource_requirements',
 228             'runtime_constraints': {
 229                 'vcpus': 3,
 230                 'ram': 3145728000,
 231                 'keep_cache_ram': 536870912,
 232                 'API': True
 233             },
 234             'use_existing': False,
 235             'priority': 500,
 236             'mounts': {
 237                 '/tmp': {'kind': 'tmp',
 238                          "capacity": 4194304000 },
 239                 '/var/spool/cwl': {'kind': 'tmp',
 240                                    "capacity": 5242880000 }
 241             },
 242             'state': 'Committed',
 243             'output_name': 'Output for step test_resource_requirements',
 244             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 245             'output_path': '/var/spool/cwl',
 246             'output_ttl': 7200,
 247             'container_image': '99999999999999999999999999999993+99',
 248             'command': ['ls'],
 249             'cwd': '/var/spool/cwl',
 250             'scheduling_parameters': {
 251                 'partitions': ['blurb']
 252             },
 253             'properties': {},
 254             'secret_mounts': {},
 255             'output_storage_classes': ["default"]
 256         }
 257
 258         call_body = call_kwargs.get('body', None)
 259         self.assertNotEqual(None, call_body)
 260         for key in call_body:
 261             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 262
 263
 264     # The test passes some fields in builder.resources
 265     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 266     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 267     @mock.patch("arvados.collection.Collection")
 268     def test_initial_work_dir(self, collection_mock, keepdocker):
 269         runner = mock.MagicMock()
 270         runner.ignore_docker_for_reuse = False
 271         runner.intermediate_output_ttl = 0
 272         runner.secret_store = cwltool.secrets.SecretStore()
 273
 274         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 275         runner.api.collections().get().execute.return_value = {
 276             "portable_data_hash": "99999999999999999999999999999993+99"}
 277
 278         sourcemock = mock.MagicMock()
 279         def get_collection_mock(p):
 280             if "/" in p:
 281                 return (sourcemock, p.split("/", 1)[1])
 282             else:
 283                 return (sourcemock, "")
 284         runner.fs_access.get_collection.side_effect = get_collection_mock
 285
 286         vwdmock = mock.MagicMock()
 287         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 288
 289         tool = cmap({
 290             "inputs": [],
 291             "outputs": [],
 292             "hints": [{
 293                 "class": "InitialWorkDirRequirement",
 294                 "listing": [{
 295                     "class": "File",
 296                     "basename": "foo",
 297                     "location": "keep:99999999999999999999999999999995+99/bar"
 298                 },
 299                 {
 300                     "class": "Directory",
 301                     "basename": "foo2",
 302                     "location": "keep:99999999999999999999999999999995+99"
 303                 },
 304                 {
 305                     "class": "File",
 306                     "basename": "filename",
 307                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 308                 },
 309                 {
 310                     "class": "Directory",
 311                     "basename": "subdir",
 312                     "location": "keep:99999999999999999999999999999995+99/subdir"
 313                 }                        ]
 314             }],
 315             "baseCommand": "ls",
 316             "id": "#",
 317             "class": "org.w3id.cwl.cwl.CommandLineTool"
 318         })
 319
 320         loadingContext, runtimeContext = self.helper(runner)
 321         runtimeContext.name = "test_initial_work_dir"
 322
 323         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 324         arvtool.formatgraph = None
 325         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 326             j.run(runtimeContext)
 327
 328         call_args, call_kwargs = runner.api.container_requests().create.call_args
 329
 330         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 332         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 333         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 334
 335         call_body_expected = {
 336             'environment': {
 337                 'HOME': '/var/spool/cwl',
 338                 'TMPDIR': '/tmp'
 339             },
 340             'name': 'test_initial_work_dir',
 341             'runtime_constraints': {
 342                 'vcpus': 1,
 343                 'ram': 1073741824
 344             },
 345             'use_existing': True,
 346             'priority': 500,
 347             'mounts': {
 348                 '/tmp': {'kind': 'tmp',
 349                          "capacity": 1073741824 },
 350                 '/var/spool/cwl': {'kind': 'tmp',
 351                                    "capacity": 1073741824 },
 352                 '/var/spool/cwl/foo': {
 353                     'kind': 'collection',
 354                     'path': 'foo',
 355                     'portable_data_hash': '99999999999999999999999999999996+99'
 356                 },
 357                 '/var/spool/cwl/foo2': {
 358                     'kind': 'collection',
 359                     'path': 'foo2',
 360                     'portable_data_hash': '99999999999999999999999999999996+99'
 361                 },
 362                 '/var/spool/cwl/filename': {
 363                     'kind': 'collection',
 364                     'path': 'filename',
 365                     'portable_data_hash': '99999999999999999999999999999996+99'
 366                 },
 367                 '/var/spool/cwl/subdir': {
 368                     'kind': 'collection',
 369                     'path': 'subdir',
 370                     'portable_data_hash': '99999999999999999999999999999996+99'
 371                 }
 372             },
 373             'state': 'Committed',
 374             'output_name': 'Output for step test_initial_work_dir',
 375             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 376             'output_path': '/var/spool/cwl',
 377             'output_ttl': 0,
 378             'container_image': '99999999999999999999999999999993+99',
 379             'command': ['ls'],
 380             'cwd': '/var/spool/cwl',
 381             'scheduling_parameters': {
 382             },
 383             'properties': {},
 384             'secret_mounts': {},
 385             'output_storage_classes': ["default"]
 386         }
 387
 388         call_body = call_kwargs.get('body', None)
 389         self.assertNotEqual(None, call_body)
 390         for key in call_body:
 391             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 392
 393
 394     # Test redirecting stdin/stdout/stderr
 395     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 396     def test_redirects(self, keepdocker):
 397         runner = mock.MagicMock()
 398         runner.ignore_docker_for_reuse = False
 399         runner.intermediate_output_ttl = 0
 400         runner.secret_store = cwltool.secrets.SecretStore()
 401
 402         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 403         runner.api.collections().get().execute.return_value = {
 404             "portable_data_hash": "99999999999999999999999999999993+99"}
 405
 406         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 407
 408         tool = cmap({
 409             "inputs": [],
 410             "outputs": [],
 411             "baseCommand": "ls",
 412             "stdout": "stdout.txt",
 413             "stderr": "stderr.txt",
 414             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 415             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 416             "id": "#",
 417             "class": "org.w3id.cwl.cwl.CommandLineTool"
 418         })
 419
 420         loadingContext, runtimeContext = self.helper(runner)
 421         runtimeContext.name = "test_run_redirect"
 422
 423         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 424         arvtool.formatgraph = None
 425         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 426             j.run(runtimeContext)
 427             runner.api.container_requests().create.assert_called_with(
 428                 body=JsonDiffMatcher({
 429                     'environment': {
 430                         'HOME': '/var/spool/cwl',
 431                         'TMPDIR': '/tmp'
 432                     },
 433                     'name': 'test_run_redirect',
 434                     'runtime_constraints': {
 435                         'vcpus': 1,
 436                         'ram': 1073741824
 437                     },
 438                     'use_existing': True,
 439                     'priority': 500,
 440                     'mounts': {
 441                         '/tmp': {'kind': 'tmp',
 442                                  "capacity": 1073741824 },
 443                         '/var/spool/cwl': {'kind': 'tmp',
 444                                            "capacity": 1073741824 },
 445                         "stderr": {
 446                             "kind": "file",
 447                             "path": "/var/spool/cwl/stderr.txt"
 448                         },
 449                         "stdin": {
 450                             "kind": "collection",
 451                             "path": "file.txt",
 452                             "portable_data_hash": "99999999999999999999999999999996+99"
 453                         },
 454                         "stdout": {
 455                             "kind": "file",
 456                             "path": "/var/spool/cwl/stdout.txt"
 457                         },
 458                     },
 459                     'state': 'Committed',
 460                     "output_name": "Output for step test_run_redirect",
 461                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 462                     'output_path': '/var/spool/cwl',
 463                     'output_ttl': 0,
 464                     'container_image': '99999999999999999999999999999993+99',
 465                     'command': ['ls', '/var/spool/cwl'],
 466                     'cwd': '/var/spool/cwl',
 467                     'scheduling_parameters': {},
 468                     'properties': {},
 469                     'secret_mounts': {},
 470                     'output_storage_classes': ["default"]
 471                 }))
 472
 473     @mock.patch("arvados.collection.Collection")
 474     def test_done(self, col):
 475         api = mock.MagicMock()
 476
 477         runner = mock.MagicMock()
 478         runner.api = api
 479         runner.num_retries = 0
 480         runner.ignore_docker_for_reuse = False
 481         runner.intermediate_output_ttl = 0
 482         runner.secret_store = cwltool.secrets.SecretStore()
 483
 484         runner.api.containers().get().execute.return_value = {"state":"Complete",
 485                                                               "output": "abc+123",
 486                                                               "exit_code": 0}
 487
 488         col().open.return_value = []
 489
 490         loadingContext, runtimeContext = self.helper(runner)
 491
 492         arvjob = arvados_cwl.ArvadosContainer(runner,
 493                                               runtimeContext,
 494                                               mock.MagicMock(),
 495                                               {},
 496                                               None,
 497                                               [],
 498                                               [],
 499                                               "testjob")
 500         arvjob.output_callback = mock.MagicMock()
 501         arvjob.collect_outputs = mock.MagicMock()
 502         arvjob.successCodes = [0]
 503         arvjob.outdir = "/var/spool/cwl"
 504         arvjob.output_ttl = 3600
 505
 506         arvjob.collect_outputs.return_value = {"out": "stuff"}
 507
 508         arvjob.done({
 509             "state": "Final",
 510             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 511             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 512             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 513             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 514             "modified_at": "2017-05-26T12:01:22Z"
 515         })
 516
 517         self.assertFalse(api.collections().create.called)
 518         self.assertFalse(runner.runtime_status_error.called)
 519
 520         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 521         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 522         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 523
 524     # Test to make sure we dont call runtime_status_update if we already did
 525     # some where higher up in the call stack
 526     @mock.patch("arvados_cwl.util.get_current_container")
 527     def test_recursive_runtime_status_update(self, gcc_mock):
 528         self.setup_and_test_container_executor_and_logging(gcc_mock)
 529         root_logger = logging.getLogger('')
 530
 531         # get_current_container is invoked when we call runtime_status_update
 532         # so try and log again!
 533         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 534         try:
 535             root_logger.error("First Error")
 536         except RuntimeError:
 537             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 538
 539
 540     # Test to make sure that an exception raised from
 541     # get_current_container doesn't cause the logger to raise an
 542     # exception
 543     @mock.patch("arvados_cwl.util.get_current_container")
 544     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 545         self.setup_and_test_container_executor_and_logging(gcc_mock)
 546         root_logger = logging.getLogger('')
 547
 548         # get_current_container is invoked when we call
 549         # runtime_status_update, it is going to also raise an
 550         # exception.
 551         gcc_mock.side_effect = Exception("Second Error")
 552         try:
 553             root_logger.error("First Error")
 554         except Exception:
 555             self.fail("Exception in logger should not propagate")
 556         self.assertTrue(gcc_mock.called)
 557
 558     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 559     @mock.patch("arvados_cwl.util.get_current_container")
 560     @mock.patch("arvados.collection.CollectionReader")
 561     @mock.patch("arvados.collection.Collection")
 562     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 563         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 564
 565         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 566         self.assertTrue(gcc_mock.called)
 567
 568         runner.num_retries = 0
 569         runner.ignore_docker_for_reuse = False
 570         runner.intermediate_output_ttl = 0
 571         runner.secret_store = cwltool.secrets.SecretStore()
 572         runner.label = mock.MagicMock()
 573         runner.label.return_value = '[container testjob]'
 574
 575         runner.api.containers().get().execute.return_value = {
 576             "state":"Complete",
 577             "output": "abc+123",
 578             "exit_code": 1,
 579             "log": "def+234"
 580         }
 581
 582         col().open.return_value = []
 583
 584         loadingContext, runtimeContext = self.helper(runner)
 585
 586         arvjob = arvados_cwl.ArvadosContainer(runner,
 587                                               runtimeContext,
 588                                               mock.MagicMock(),
 589                                               {},
 590                                               None,
 591                                               [],
 592                                               [],
 593                                               "testjob")
 594         arvjob.output_callback = mock.MagicMock()
 595         arvjob.collect_outputs = mock.MagicMock()
 596         arvjob.successCodes = [0]
 597         arvjob.outdir = "/var/spool/cwl"
 598         arvjob.output_ttl = 3600
 599         arvjob.collect_outputs.return_value = {"out": "stuff"}
 600
 601         arvjob.done({
 602             "state": "Final",
 603             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 604             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 605             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 606             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 607             "modified_at": "2017-05-26T12:01:22Z"
 608         })
 609
 610         rts_mock.assert_called_with(
 611             'error',
 612             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 613             '  ** log is empty **'
 614         )
 615         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 616
 617     # The test passes no builder.resources
 618     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 619     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 620     def test_mounts(self, keepdocker):
 621         runner = mock.MagicMock()
 622         runner.ignore_docker_for_reuse = False
 623         runner.intermediate_output_ttl = 0
 624         runner.secret_store = cwltool.secrets.SecretStore()
 625
 626         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 627         runner.api.collections().get().execute.return_value = {
 628             "portable_data_hash": "99999999999999999999999999999994+99",
 629             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 630
 631         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 632
 633         tool = cmap({
 634             "inputs": [
 635                 {"id": "p1",
 636                  "type": "Directory"}
 637             ],
 638             "outputs": [],
 639             "baseCommand": "ls",
 640             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 641             "id": "#",
 642             "class": "org.w3id.cwl.cwl.CommandLineTool"
 643         })
 644
 645         loadingContext, runtimeContext = self.helper(runner)
 646         runtimeContext.name = "test_run_mounts"
 647
 648         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 649         arvtool.formatgraph = None
 650         job_order = {
 651             "p1": {
 652                 "class": "Directory",
 653                 "location": "keep:99999999999999999999999999999994+44",
 654                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 655                 "listing": [
 656                     {
 657                         "class": "File",
 658                         "location": "keep:99999999999999999999999999999994+44/file1",
 659                     },
 660                     {
 661                         "class": "File",
 662                         "location": "keep:99999999999999999999999999999994+44/file2",
 663                     }
 664                 ]
 665             }
 666         }
 667         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 668             j.run(runtimeContext)
 669             runner.api.container_requests().create.assert_called_with(
 670                 body=JsonDiffMatcher({
 671                     'environment': {
 672                         'HOME': '/var/spool/cwl',
 673                         'TMPDIR': '/tmp'
 674                     },
 675                     'name': 'test_run_mounts',
 676                     'runtime_constraints': {
 677                         'vcpus': 1,
 678                         'ram': 1073741824
 679                     },
 680                     'use_existing': True,
 681                     'priority': 500,
 682                     'mounts': {
 683                         "/keep/99999999999999999999999999999994+44": {
 684                             "kind": "collection",
 685                             "portable_data_hash": "99999999999999999999999999999994+44",
 686                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 687                         },
 688                         '/tmp': {'kind': 'tmp',
 689                                  "capacity": 1073741824 },
 690                         '/var/spool/cwl': {'kind': 'tmp',
 691                                            "capacity": 1073741824 }
 692                     },
 693                     'state': 'Committed',
 694                     'output_name': 'Output for step test_run_mounts',
 695                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 696                     'output_path': '/var/spool/cwl',
 697                     'output_ttl': 0,
 698                     'container_image': '99999999999999999999999999999994+99',
 699                     'command': ['ls', '/var/spool/cwl'],
 700                     'cwd': '/var/spool/cwl',
 701                     'scheduling_parameters': {},
 702                     'properties': {},
 703                     'secret_mounts': {},
 704                     'output_storage_classes': ["default"]
 705                 }))
 706
 707     # The test passes no builder.resources
 708     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 709     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 710     def test_secrets(self, keepdocker):
 711         runner = mock.MagicMock()
 712         runner.ignore_docker_for_reuse = False
 713         runner.intermediate_output_ttl = 0
 714         runner.secret_store = cwltool.secrets.SecretStore()
 715
 716         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 717         runner.api.collections().get().execute.return_value = {
 718             "portable_data_hash": "99999999999999999999999999999993+99"}
 719
 720         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 721
 722         tool = cmap({"arguments": ["md5sum", "example.conf"],
 723                      "class": "org.w3id.cwl.cwl.CommandLineTool",
 724                      "hints": [
 725                          {
 726                              "class": "http://commonwl.org/cwltool#Secrets",
 727                              "secrets": [
 728                                  "#secret_job.cwl/pw"
 729                              ]
 730                          }
 731                      ],
 732                      "id": "#secret_job.cwl",
 733                      "inputs": [
 734                          {
 735                              "id": "#secret_job.cwl/pw",
 736                              "type": "string"
 737                          }
 738                      ],
 739                      "outputs": [
 740                      ],
 741                      "requirements": [
 742                          {
 743                              "class": "InitialWorkDirRequirement",
 744                              "listing": [
 745                                  {
 746                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 747                                      "entryname": "example.conf"
 748                                  }
 749                              ]
 750                          }
 751                      ]})
 752
 753         loadingContext, runtimeContext = self.helper(runner)
 754         runtimeContext.name = "test_secrets"
 755
 756         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 757         arvtool.formatgraph = None
 758
 759         job_order = {"pw": "blorp"}
 760         runner.secret_store.store(["pw"], job_order)
 761
 762         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 763             j.run(runtimeContext)
 764             runner.api.container_requests().create.assert_called_with(
 765                 body=JsonDiffMatcher({
 766                     'environment': {
 767                         'HOME': '/var/spool/cwl',
 768                         'TMPDIR': '/tmp'
 769                     },
 770                     'name': 'test_secrets',
 771                     'runtime_constraints': {
 772                         'vcpus': 1,
 773                         'ram': 1073741824
 774                     },
 775                     'use_existing': True,
 776                     'priority': 500,
 777                     'mounts': {
 778                         '/tmp': {'kind': 'tmp',
 779                                  "capacity": 1073741824
 780                              },
 781                         '/var/spool/cwl': {'kind': 'tmp',
 782                                            "capacity": 1073741824 }
 783                     },
 784                     'state': 'Committed',
 785                     'output_name': 'Output for step test_secrets',
 786                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 787                     'output_path': '/var/spool/cwl',
 788                     'output_ttl': 0,
 789                     'container_image': '99999999999999999999999999999993+99',
 790                     'command': ['md5sum', 'example.conf'],
 791                     'cwd': '/var/spool/cwl',
 792                     'scheduling_parameters': {},
 793                     'properties': {},
 794                     "secret_mounts": {
 795                         "/var/spool/cwl/example.conf": {
 796                             "content": "username: user\npassword: blorp\n",
 797                             "kind": "text"
 798                         }
 799                     },
 800                     'output_storage_classes': ["default"]
 801                 }))
 802
 803     # The test passes no builder.resources
 804     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 805     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 806     def test_timelimit(self, keepdocker):
 807         runner = mock.MagicMock()
 808         runner.ignore_docker_for_reuse = False
 809         runner.intermediate_output_ttl = 0
 810         runner.secret_store = cwltool.secrets.SecretStore()
 811
 812         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 813         runner.api.collections().get().execute.return_value = {
 814             "portable_data_hash": "99999999999999999999999999999993+99"}
 815
 816         tool = cmap({
 817             "inputs": [],
 818             "outputs": [],
 819             "baseCommand": "ls",
 820             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 821             "id": "#",
 822             "class": "org.w3id.cwl.cwl.CommandLineTool",
 823             "hints": [
 824                 {
 825                     "class": "ToolTimeLimit",
 826                     "timelimit": 42
 827                 }
 828             ]
 829         })
 830
 831         loadingContext, runtimeContext = self.helper(runner)
 832         runtimeContext.name = "test_timelimit"
 833
 834         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 835         arvtool.formatgraph = None
 836
 837         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 838             j.run(runtimeContext)
 839
 840         _, kwargs = runner.api.container_requests().create.call_args
 841         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 842
 843
 844     # The test passes no builder.resources
 845     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 846     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 847     def test_setting_storage_class(self, keepdocker):
 848         arv_docker_clear_cache()
 849
 850         runner = mock.MagicMock()
 851         runner.ignore_docker_for_reuse = False
 852         runner.intermediate_output_ttl = 0
 853         runner.secret_store = cwltool.secrets.SecretStore()
 854
 855         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 856         runner.api.collections().get().execute.return_value = {
 857             "portable_data_hash": "99999999999999999999999999999993+99"}
 858
 859         tool = cmap({
 860             "inputs": [],
 861             "outputs": [],
 862             "baseCommand": "ls",
 863             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 864             "id": "#",
 865             "class": "org.w3id.cwl.cwl.CommandLineTool",
 866             "hints": [
 867                 {
 868                     "class": "http://arvados.org/cwl#OutputStorageClass",
 869                     "finalStorageClass": ["baz_sc", "qux_sc"],
 870                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 871                 }
 872             ]
 873         })
 874
 875         loadingContext, runtimeContext = self.helper(runner, True)
 876
 877         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 878         arvtool.formatgraph = None
 879
 880         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 881             j.run(runtimeContext)
 882             runner.api.container_requests().create.assert_called_with(
 883                 body=JsonDiffMatcher({
 884                     'environment': {
 885                         'HOME': '/var/spool/cwl',
 886                         'TMPDIR': '/tmp'
 887                     },
 888                     'name': 'test_run_True',
 889                     'runtime_constraints': {
 890                         'vcpus': 1,
 891                         'ram': 1073741824
 892                     },
 893                     'use_existing': True,
 894                     'priority': 500,
 895                     'mounts': {
 896                         '/tmp': {'kind': 'tmp',
 897                                  "capacity": 1073741824
 898                              },
 899                         '/var/spool/cwl': {'kind': 'tmp',
 900                                            "capacity": 1073741824 }
 901                     },
 902                     'state': 'Committed',
 903                     'output_name': 'Output for step test_run_True',
 904                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 905                     'output_path': '/var/spool/cwl',
 906                     'output_ttl': 0,
 907                     'container_image': '99999999999999999999999999999993+99',
 908                     'command': ['ls', '/var/spool/cwl'],
 909                     'cwd': '/var/spool/cwl',
 910                     'scheduling_parameters': {},
 911                     'properties': {},
 912                     'secret_mounts': {},
 913                     'output_storage_classes': ["foo_sc", "bar_sc"]
 914                 }))
 915
 916
 917     # The test passes no builder.resources
 918     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 919     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 920     def test_setting_process_properties(self, keepdocker):
 921         arv_docker_clear_cache()
 922
 923         runner = mock.MagicMock()
 924         runner.ignore_docker_for_reuse = False
 925         runner.intermediate_output_ttl = 0
 926         runner.secret_store = cwltool.secrets.SecretStore()
 927
 928         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 929         runner.api.collections().get().execute.return_value = {
 930             "portable_data_hash": "99999999999999999999999999999993+99"}
 931
 932         tool = cmap({
 933             "inputs": [
 934                 {"id": "x", "type": "string"}],
 935             "outputs": [],
 936             "baseCommand": "ls",
 937             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 938             "id": "#",
 939             "class": "org.w3id.cwl.cwl.CommandLineTool",
 940             "hints": [
 941             {
 942                 "class": "http://arvados.org/cwl#ProcessProperties",
 943                 "processProperties": [
 944                     {"propertyName": "foo",
 945                      "propertyValue": "bar"},
 946                     {"propertyName": "baz",
 947                      "propertyValue": "$(inputs.x)"},
 948                     {"propertyName": "quux",
 949                      "propertyValue": {
 950                          "q1": 1,
 951                          "q2": 2
 952                      }
 953                     }
 954                 ],
 955             }
 956         ]
 957         })
 958
 959         loadingContext, runtimeContext = self.helper(runner, True)
 960
 961         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 962         arvtool.formatgraph = None
 963
 964         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 965             j.run(runtimeContext)
 966             runner.api.container_requests().create.assert_called_with(
 967                 body=JsonDiffMatcher({
 968                     'environment': {
 969                         'HOME': '/var/spool/cwl',
 970                         'TMPDIR': '/tmp'
 971                     },
 972                     'name': 'test_run_True',
 973                     'runtime_constraints': {
 974                         'vcpus': 1,
 975                         'ram': 1073741824
 976                     },
 977                     'use_existing': True,
 978                     'priority': 500,
 979                     'mounts': {
 980                         '/tmp': {'kind': 'tmp',
 981                                  "capacity": 1073741824
 982                              },
 983                         '/var/spool/cwl': {'kind': 'tmp',
 984                                            "capacity": 1073741824 }
 985                     },
 986                     'state': 'Committed',
 987                     'output_name': 'Output for step test_run_True',
 988                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 989                     'output_path': '/var/spool/cwl',
 990                     'output_ttl': 0,
 991                     'container_image': '99999999999999999999999999999993+99',
 992                     'command': ['ls', '/var/spool/cwl'],
 993                     'cwd': '/var/spool/cwl',
 994                     'scheduling_parameters': {},
 995                     'properties': {
 996                         "baz": "blorp",
 997                         "foo": "bar",
 998                         "quux": {
 999                             "q1": 1,
1000                             "q2": 2
1001                         }
1002                     },
1003                     'secret_mounts': {},
1004                     'output_storage_classes': ["default"]
1005                 }))
1006
1007
1008 class TestWorkflow(unittest.TestCase):
1009     def setUp(self):
1010         cwltool.process._names = set()
1011         arv_docker_clear_cache()
1012
1013     def helper(self, runner, enable_reuse=True):
1014         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1015
1016         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1017                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1018
1019         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1020         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1021         document_loader.fetch_text = document_loader.fetcher.fetch_text
1022         document_loader.check_exists = document_loader.fetcher.check_exists
1023
1024         loadingContext = arvados_cwl.context.ArvLoadingContext(
1025             {"avsc_names": avsc_names,
1026              "basedir": "",
1027              "make_fs_access": make_fs_access,
1028              "loader": document_loader,
1029              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1030              "construct_tool_object": runner.arv_make_tool})
1031         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1032             {"work_api": "containers",
1033              "basedir": "",
1034              "name": "test_run_wf_"+str(enable_reuse),
1035              "make_fs_access": make_fs_access,
1036              "tmpdir": "/tmp",
1037              "enable_reuse": enable_reuse,
1038              "priority": 500})
1039
1040         return loadingContext, runtimeContext
1041
1042     # The test passes no builder.resources
1043     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1044     @mock.patch("arvados.collection.CollectionReader")
1045     @mock.patch("arvados.collection.Collection")
1046     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1047     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1048         arvados_cwl.add_arv_hints()
1049
1050         api = mock.MagicMock()
1051         api._rootDesc = get_rootDesc()
1052
1053         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1054         self.assertEqual(runner.work_api, 'containers')
1055
1056         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1057         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1058         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1059                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1060
1061         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1062         runner.ignore_docker_for_reuse = False
1063         runner.num_retries = 0
1064         runner.secret_store = cwltool.secrets.SecretStore()
1065
1066         loadingContext, runtimeContext = self.helper(runner)
1067         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1068
1069         mockcollectionreader().exists.return_value = True
1070
1071         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1072         metadata["cwlVersion"] = tool["cwlVersion"]
1073
1074         mockc = mock.MagicMock()
1075         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1076         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1077
1078         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1079         arvtool.formatgraph = None
1080         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1081
1082         next(it).run(runtimeContext)
1083         next(it).run(runtimeContext)
1084
1085         with open("tests/wf/scatter2_subwf.cwl") as f:
1086             subwf = StripYAMLComments(f.read()).rstrip()
1087
1088         runner.api.container_requests().create.assert_called_with(
1089             body=JsonDiffMatcher({
1090                 "command": [
1091                     "cwltool",
1092                     "--no-container",
1093                     "--move-outputs",
1094                     "--preserve-entire-environment",
1095                     "workflow.cwl",
1096                     "cwl.input.yml"
1097                 ],
1098                 "container_image": "99999999999999999999999999999993+99",
1099                 "cwd": "/var/spool/cwl",
1100                 "environment": {
1101                     "HOME": "/var/spool/cwl",
1102                     "TMPDIR": "/tmp"
1103                 },
1104                 "mounts": {
1105                     "/keep/99999999999999999999999999999999+118": {
1106                         "kind": "collection",
1107                         "portable_data_hash": "99999999999999999999999999999999+118"
1108                     },
1109                     "/tmp": {
1110                         "capacity": 1073741824,
1111                         "kind": "tmp"
1112                     },
1113                     "/var/spool/cwl": {
1114                         "capacity": 1073741824,
1115                         "kind": "tmp"
1116                     },
1117                     "/var/spool/cwl/cwl.input.yml": {
1118                         "kind": "collection",
1119                         "path": "cwl.input.yml",
1120                         "portable_data_hash": "99999999999999999999999999999996+99"
1121                     },
1122                     "/var/spool/cwl/workflow.cwl": {
1123                         "kind": "collection",
1124                         "path": "workflow.cwl",
1125                         "portable_data_hash": "99999999999999999999999999999996+99"
1126                     },
1127                     "stdout": {
1128                         "kind": "file",
1129                         "path": "/var/spool/cwl/cwl.output.json"
1130                     }
1131                 },
1132                 "name": "scatterstep",
1133                 "output_name": "Output for step scatterstep",
1134                 "output_path": "/var/spool/cwl",
1135                 "output_ttl": 0,
1136                 "priority": 500,
1137                 "properties": {},
1138                 "runtime_constraints": {
1139                     "ram": 1073741824,
1140                     "vcpus": 1
1141                 },
1142                 "scheduling_parameters": {},
1143                 "secret_mounts": {},
1144                 "state": "Committed",
1145                 "use_existing": True,
1146                 'output_storage_classes': ["default"]
1147             }))
1148         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1149         mockc.open().__enter__().write.assert_has_calls([mock.call(
1150 '''{
1151   "fileblub": {
1152     "basename": "token.txt",
1153     "class": "File",
1154     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1155     "size": 0
1156   },
1157   "sleeptime": 5
1158 }''')])
1159
1160     # The test passes no builder.resources
1161     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1162     @mock.patch("arvados.collection.CollectionReader")
1163     @mock.patch("arvados.collection.Collection")
1164     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1165     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1166         arvados_cwl.add_arv_hints()
1167
1168         api = mock.MagicMock()
1169         api._rootDesc = get_rootDesc()
1170
1171         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1172         self.assertEqual(runner.work_api, 'containers')
1173
1174         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1175         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1176                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1177         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1178                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1179
1180         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1181         runner.ignore_docker_for_reuse = False
1182         runner.num_retries = 0
1183         runner.secret_store = cwltool.secrets.SecretStore()
1184
1185         loadingContext, runtimeContext = self.helper(runner)
1186         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1187         loadingContext.do_update = True
1188         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1189
1190         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1191
1192         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1193         arvtool.formatgraph = None
1194         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1195
1196         next(it).run(runtimeContext)
1197         next(it).run(runtimeContext)
1198
1199         with open("tests/wf/echo-subwf.cwl") as f:
1200             subwf = StripYAMLComments(f.read())
1201
1202         runner.api.container_requests().create.assert_called_with(
1203             body=JsonDiffMatcher({
1204                 'output_ttl': 0,
1205                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1206                 'scheduling_parameters': {},
1207                 'name': u'echo-subwf',
1208                 'secret_mounts': {},
1209                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1210                 'properties': {},
1211                 'priority': 500,
1212                 'mounts': {
1213                     '/var/spool/cwl/cwl.input.yml': {
1214                         'portable_data_hash': '99999999999999999999999999999996+99',
1215                         'kind': 'collection',
1216                         'path': 'cwl.input.yml'
1217                     },
1218                     '/var/spool/cwl/workflow.cwl': {
1219                         'portable_data_hash': '99999999999999999999999999999996+99',
1220                         'kind': 'collection',
1221                         'path': 'workflow.cwl'
1222                     },
1223                     'stdout': {
1224                         'path': '/var/spool/cwl/cwl.output.json',
1225                         'kind': 'file'
1226                     },
1227                     '/tmp': {
1228                         'kind': 'tmp',
1229                         'capacity': 1073741824
1230                     }, '/var/spool/cwl': {
1231                         'kind': 'tmp',
1232                         'capacity': 3221225472
1233                     }
1234                 },
1235                 'state': 'Committed',
1236                 'output_path': '/var/spool/cwl',
1237                 'container_image': '99999999999999999999999999999993+99',
1238                 'command': [
1239                     u'cwltool',
1240                     u'--no-container',
1241                     u'--move-outputs',
1242                     u'--preserve-entire-environment',
1243                     u'workflow.cwl',
1244                     u'cwl.input.yml'
1245                 ],
1246                 'use_existing': True,
1247                 'output_name': u'Output for step echo-subwf',
1248                 'cwd': '/var/spool/cwl',
1249                 'output_storage_classes': ["default"]
1250             }))
1251
1252     def test_default_work_api(self):
1253         arvados_cwl.add_arv_hints()
1254
1255         api = mock.MagicMock()
1256         api._rootDesc = copy.deepcopy(get_rootDesc())
1257         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1258         self.assertEqual(runner.work_api, 'containers')