sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from cwltool.update import INTERNAL_VERSION
  22 from schema_salad.ref_resolver import Loader
  23 from schema_salad.sourceline import cmap
  24
  25 from .matcher import JsonDiffMatcher, StripYAMLComments
  26 from .mock_discovery import get_rootDesc
  27
  28 if not os.getenv('ARVADOS_DEBUG'):
  29     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  30     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  31
  32 class CollectionMock(object):
  33     def __init__(self, vwdmock, *args, **kwargs):
  34         self.vwdmock = vwdmock
  35         self.count = 0
  36
  37     def open(self, *args, **kwargs):
  38         self.count += 1
  39         return self.vwdmock.open(*args, **kwargs)
  40
  41     def copy(self, *args, **kwargs):
  42         self.count += 1
  43         self.vwdmock.copy(*args, **kwargs)
  44
  45     def save_new(self, *args, **kwargs):
  46         pass
  47
  48     def __len__(self):
  49         return self.count
  50
  51     def portable_data_hash(self):
  52         if self.count == 0:
  53             return arvados.config.EMPTY_BLOCK_LOCATOR
  54         else:
  55             return "99999999999999999999999999999996+99"
  56
  57
  58 class TestContainer(unittest.TestCase):
  59
  60     def setUp(self):
  61         cwltool.process._names = set()
  62         arv_docker_clear_cache()
  63
  64     def helper(self, runner, enable_reuse=True):
  65         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  66
  67         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  68                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  69         loadingContext = arvados_cwl.context.ArvLoadingContext(
  70             {"avsc_names": avsc_names,
  71              "basedir": "",
  72              "make_fs_access": make_fs_access,
  73              "loader": Loader({}),
  74              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  75         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  76             {"work_api": "containers",
  77              "basedir": "",
  78              "name": "test_run_"+str(enable_reuse),
  79              "make_fs_access": make_fs_access,
  80              "tmpdir": "/tmp",
  81              "enable_reuse": enable_reuse,
  82              "priority": 500,
  83              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84             })
  85
  86         return loadingContext, runtimeContext
  87
  88     # Helper function to set up the ArvCwlExecutor to use the containers api
  89     # and test that the RuntimeStatusLoggingHandler is set up correctly
  90     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  91         api = mock.MagicMock()
  92         api._rootDesc = copy.deepcopy(get_rootDesc())
  93
  94         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  95         # adds the logging handler that will call runtime_status_update() mock
  96         self.assertFalse(gcc_mock.called)
  97         runner = arvados_cwl.ArvCwlExecutor(api)
  98         self.assertEqual(runner.work_api, 'containers')
  99         root_logger = logging.getLogger('')
 100         handlerClasses = [h.__class__ for h in root_logger.handlers]
 101         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 102         return runner
 103
 104     # The test passes no builder.resources
 105     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 106     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 107     def test_run(self, keepdocker):
 108         for enable_reuse in (True, False):
 109             arv_docker_clear_cache()
 110
 111             runner = mock.MagicMock()
 112             runner.ignore_docker_for_reuse = False
 113             runner.intermediate_output_ttl = 0
 114             runner.secret_store = cwltool.secrets.SecretStore()
 115
 116             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 117             runner.api.collections().get().execute.return_value = {
 118                 "portable_data_hash": "99999999999999999999999999999993+99"}
 119
 120             tool = cmap({
 121                 "inputs": [],
 122                 "outputs": [],
 123                 "baseCommand": "ls",
 124                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 125                 "id": "#",
 126                 "class": "CommandLineTool"
 127             })
 128
 129             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 130
 131             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 132             arvtool.formatgraph = None
 133
 134             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 135                 j.run(runtimeContext)
 136                 runner.api.container_requests().create.assert_called_with(
 137                     body=JsonDiffMatcher({
 138                         'environment': {
 139                             'HOME': '/var/spool/cwl',
 140                             'TMPDIR': '/tmp'
 141                         },
 142                         'name': 'test_run_'+str(enable_reuse),
 143                         'runtime_constraints': {
 144                             'vcpus': 1,
 145                             'ram': 1073741824
 146                         },
 147                         'use_existing': enable_reuse,
 148                         'priority': 500,
 149                         'mounts': {
 150                             '/tmp': {'kind': 'tmp',
 151                                      "capacity": 1073741824
 152                                  },
 153                             '/var/spool/cwl': {'kind': 'tmp',
 154                                                "capacity": 1073741824 }
 155                         },
 156                         'state': 'Committed',
 157                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 158                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 159                         'output_path': '/var/spool/cwl',
 160                         'output_ttl': 0,
 161                         'container_image': '99999999999999999999999999999993+99',
 162                         'command': ['ls', '/var/spool/cwl'],
 163                         'cwd': '/var/spool/cwl',
 164                         'scheduling_parameters': {},
 165                         'properties': {},
 166                         'secret_mounts': {},
 167                         'output_storage_classes': ["default"]
 168                     }))
 169
 170     # The test passes some fields in builder.resources
 171     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 172     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 173     def test_resource_requirements(self, keepdocker):
 174         runner = mock.MagicMock()
 175         runner.ignore_docker_for_reuse = False
 176         runner.intermediate_output_ttl = 3600
 177         runner.secret_store = cwltool.secrets.SecretStore()
 178
 179         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 180         runner.api.collections().get().execute.return_value = {
 181             "portable_data_hash": "99999999999999999999999999999993+99"}
 182
 183         tool = cmap({
 184             "inputs": [],
 185             "outputs": [],
 186             "hints": [{
 187                 "class": "ResourceRequirement",
 188                 "coresMin": 3,
 189                 "ramMin": 3000,
 190                 "tmpdirMin": 4000,
 191                 "outdirMin": 5000
 192             }, {
 193                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 194                 "keep_cache": 512
 195             }, {
 196                 "class": "http://arvados.org/cwl#APIRequirement",
 197             }, {
 198                 "class": "http://arvados.org/cwl#PartitionRequirement",
 199                 "partition": "blurb"
 200             }, {
 201                 "class": "http://arvados.org/cwl#IntermediateOutput",
 202                 "outputTTL": 7200
 203             }, {
 204                 "class": "http://arvados.org/cwl#ReuseRequirement",
 205                 "enableReuse": False
 206             }],
 207             "baseCommand": "ls",
 208             "id": "#",
 209             "class": "CommandLineTool"
 210         })
 211
 212         loadingContext, runtimeContext = self.helper(runner)
 213         runtimeContext.name = "test_resource_requirements"
 214
 215         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 216         arvtool.formatgraph = None
 217         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 218             j.run(runtimeContext)
 219
 220         call_args, call_kwargs = runner.api.container_requests().create.call_args
 221
 222         call_body_expected = {
 223             'environment': {
 224                 'HOME': '/var/spool/cwl',
 225                 'TMPDIR': '/tmp'
 226             },
 227             'name': 'test_resource_requirements',
 228             'runtime_constraints': {
 229                 'vcpus': 3,
 230                 'ram': 3145728000,
 231                 'keep_cache_ram': 536870912,
 232                 'API': True
 233             },
 234             'use_existing': False,
 235             'priority': 500,
 236             'mounts': {
 237                 '/tmp': {'kind': 'tmp',
 238                          "capacity": 4194304000 },
 239                 '/var/spool/cwl': {'kind': 'tmp',
 240                                    "capacity": 5242880000 }
 241             },
 242             'state': 'Committed',
 243             'output_name': 'Output for step test_resource_requirements',
 244             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 245             'output_path': '/var/spool/cwl',
 246             'output_ttl': 7200,
 247             'container_image': '99999999999999999999999999999993+99',
 248             'command': ['ls'],
 249             'cwd': '/var/spool/cwl',
 250             'scheduling_parameters': {
 251                 'partitions': ['blurb']
 252             },
 253             'properties': {},
 254             'secret_mounts': {},
 255             'output_storage_classes': ["default"]
 256         }
 257
 258         call_body = call_kwargs.get('body', None)
 259         self.assertNotEqual(None, call_body)
 260         for key in call_body:
 261             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 262
 263
 264     # The test passes some fields in builder.resources
 265     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 266     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 267     @mock.patch("arvados.collection.Collection")
 268     def test_initial_work_dir(self, collection_mock, keepdocker):
 269         runner = mock.MagicMock()
 270         runner.ignore_docker_for_reuse = False
 271         runner.intermediate_output_ttl = 0
 272         runner.secret_store = cwltool.secrets.SecretStore()
 273
 274         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 275         runner.api.collections().get().execute.return_value = {
 276             "portable_data_hash": "99999999999999999999999999999993+99"}
 277
 278         sourcemock = mock.MagicMock()
 279         def get_collection_mock(p):
 280             if "/" in p:
 281                 return (sourcemock, p.split("/", 1)[1])
 282             else:
 283                 return (sourcemock, "")
 284         runner.fs_access.get_collection.side_effect = get_collection_mock
 285
 286         vwdmock = mock.MagicMock()
 287         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 288
 289         tool = cmap({
 290             "inputs": [],
 291             "outputs": [],
 292             "hints": [{
 293                 "class": "InitialWorkDirRequirement",
 294                 "listing": [{
 295                     "class": "File",
 296                     "basename": "foo",
 297                     "location": "keep:99999999999999999999999999999995+99/bar"
 298                 },
 299                 {
 300                     "class": "Directory",
 301                     "basename": "foo2",
 302                     "location": "keep:99999999999999999999999999999995+99"
 303                 },
 304                 {
 305                     "class": "File",
 306                     "basename": "filename",
 307                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 308                 },
 309                 {
 310                     "class": "Directory",
 311                     "basename": "subdir",
 312                     "location": "keep:99999999999999999999999999999995+99/subdir"
 313                 }                        ]
 314             }],
 315             "baseCommand": "ls",
 316             "id": "#",
 317             "class": "CommandLineTool"
 318         })
 319
 320         loadingContext, runtimeContext = self.helper(runner)
 321         runtimeContext.name = "test_initial_work_dir"
 322
 323         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 324         arvtool.formatgraph = None
 325         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 326             j.run(runtimeContext)
 327
 328         call_args, call_kwargs = runner.api.container_requests().create.call_args
 329
 330         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 332         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 333         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 334
 335         call_body_expected = {
 336             'environment': {
 337                 'HOME': '/var/spool/cwl',
 338                 'TMPDIR': '/tmp'
 339             },
 340             'name': 'test_initial_work_dir',
 341             'runtime_constraints': {
 342                 'vcpus': 1,
 343                 'ram': 1073741824
 344             },
 345             'use_existing': True,
 346             'priority': 500,
 347             'mounts': {
 348                 '/tmp': {'kind': 'tmp',
 349                          "capacity": 1073741824 },
 350                 '/var/spool/cwl': {'kind': 'tmp',
 351                                    "capacity": 1073741824 },
 352                 '/var/spool/cwl/foo': {
 353                     'kind': 'collection',
 354                     'path': 'foo',
 355                     'portable_data_hash': '99999999999999999999999999999996+99'
 356                 },
 357                 '/var/spool/cwl/foo2': {
 358                     'kind': 'collection',
 359                     'path': 'foo2',
 360                     'portable_data_hash': '99999999999999999999999999999996+99'
 361                 },
 362                 '/var/spool/cwl/filename': {
 363                     'kind': 'collection',
 364                     'path': 'filename',
 365                     'portable_data_hash': '99999999999999999999999999999996+99'
 366                 },
 367                 '/var/spool/cwl/subdir': {
 368                     'kind': 'collection',
 369                     'path': 'subdir',
 370                     'portable_data_hash': '99999999999999999999999999999996+99'
 371                 }
 372             },
 373             'state': 'Committed',
 374             'output_name': 'Output for step test_initial_work_dir',
 375             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 376             'output_path': '/var/spool/cwl',
 377             'output_ttl': 0,
 378             'container_image': '99999999999999999999999999999993+99',
 379             'command': ['ls'],
 380             'cwd': '/var/spool/cwl',
 381             'scheduling_parameters': {
 382             },
 383             'properties': {},
 384             'secret_mounts': {},
 385             'output_storage_classes': ["default"]
 386         }
 387
 388         call_body = call_kwargs.get('body', None)
 389         self.assertNotEqual(None, call_body)
 390         for key in call_body:
 391             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 392
 393
 394     # Test redirecting stdin/stdout/stderr
 395     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 396     def test_redirects(self, keepdocker):
 397         runner = mock.MagicMock()
 398         runner.ignore_docker_for_reuse = False
 399         runner.intermediate_output_ttl = 0
 400         runner.secret_store = cwltool.secrets.SecretStore()
 401
 402         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 403         runner.api.collections().get().execute.return_value = {
 404             "portable_data_hash": "99999999999999999999999999999993+99"}
 405
 406         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 407
 408         tool = cmap({
 409             "inputs": [],
 410             "outputs": [],
 411             "baseCommand": "ls",
 412             "stdout": "stdout.txt",
 413             "stderr": "stderr.txt",
 414             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 415             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 416             "id": "#",
 417             "class": "CommandLineTool"
 418         })
 419
 420         loadingContext, runtimeContext = self.helper(runner)
 421         runtimeContext.name = "test_run_redirect"
 422
 423         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 424         arvtool.formatgraph = None
 425         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 426             j.run(runtimeContext)
 427             runner.api.container_requests().create.assert_called_with(
 428                 body=JsonDiffMatcher({
 429                     'environment': {
 430                         'HOME': '/var/spool/cwl',
 431                         'TMPDIR': '/tmp'
 432                     },
 433                     'name': 'test_run_redirect',
 434                     'runtime_constraints': {
 435                         'vcpus': 1,
 436                         'ram': 1073741824
 437                     },
 438                     'use_existing': True,
 439                     'priority': 500,
 440                     'mounts': {
 441                         '/tmp': {'kind': 'tmp',
 442                                  "capacity": 1073741824 },
 443                         '/var/spool/cwl': {'kind': 'tmp',
 444                                            "capacity": 1073741824 },
 445                         "stderr": {
 446                             "kind": "file",
 447                             "path": "/var/spool/cwl/stderr.txt"
 448                         },
 449                         "stdin": {
 450                             "kind": "collection",
 451                             "path": "file.txt",
 452                             "portable_data_hash": "99999999999999999999999999999996+99"
 453                         },
 454                         "stdout": {
 455                             "kind": "file",
 456                             "path": "/var/spool/cwl/stdout.txt"
 457                         },
 458                     },
 459                     'state': 'Committed',
 460                     "output_name": "Output for step test_run_redirect",
 461                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 462                     'output_path': '/var/spool/cwl',
 463                     'output_ttl': 0,
 464                     'container_image': '99999999999999999999999999999993+99',
 465                     'command': ['ls', '/var/spool/cwl'],
 466                     'cwd': '/var/spool/cwl',
 467                     'scheduling_parameters': {},
 468                     'properties': {},
 469                     'secret_mounts': {},
 470                     'output_storage_classes': ["default"]
 471                 }))
 472
 473     @mock.patch("arvados.collection.Collection")
 474     def test_done(self, col):
 475         api = mock.MagicMock()
 476
 477         runner = mock.MagicMock()
 478         runner.api = api
 479         runner.num_retries = 0
 480         runner.ignore_docker_for_reuse = False
 481         runner.intermediate_output_ttl = 0
 482         runner.secret_store = cwltool.secrets.SecretStore()
 483
 484         runner.api.containers().get().execute.return_value = {"state":"Complete",
 485                                                               "output": "abc+123",
 486                                                               "exit_code": 0}
 487
 488         col().open.return_value = []
 489
 490         loadingContext, runtimeContext = self.helper(runner)
 491
 492         arvjob = arvados_cwl.ArvadosContainer(runner,
 493                                               runtimeContext,
 494                                               mock.MagicMock(),
 495                                               {},
 496                                               None,
 497                                               [],
 498                                               [],
 499                                               "testjob")
 500         arvjob.output_callback = mock.MagicMock()
 501         arvjob.collect_outputs = mock.MagicMock()
 502         arvjob.successCodes = [0]
 503         arvjob.outdir = "/var/spool/cwl"
 504         arvjob.output_ttl = 3600
 505
 506         arvjob.collect_outputs.return_value = {"out": "stuff"}
 507
 508         arvjob.done({
 509             "state": "Final",
 510             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 511             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 512             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 513             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 514             "modified_at": "2017-05-26T12:01:22Z"
 515         })
 516
 517         self.assertFalse(api.collections().create.called)
 518         self.assertFalse(runner.runtime_status_error.called)
 519
 520         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 521         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 522         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 523
 524     # Test to make sure we dont call runtime_status_update if we already did
 525     # some where higher up in the call stack
 526     @mock.patch("arvados_cwl.util.get_current_container")
 527     def test_recursive_runtime_status_update(self, gcc_mock):
 528         self.setup_and_test_container_executor_and_logging(gcc_mock)
 529         root_logger = logging.getLogger('')
 530
 531         # get_current_container is invoked when we call runtime_status_update
 532         # so try and log again!
 533         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 534         try:
 535             root_logger.error("First Error")
 536         except RuntimeError:
 537             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 538
 539
 540     # Test to make sure that an exception raised from
 541     # get_current_container doesn't cause the logger to raise an
 542     # exception
 543     @mock.patch("arvados_cwl.util.get_current_container")
 544     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 545         self.setup_and_test_container_executor_and_logging(gcc_mock)
 546         root_logger = logging.getLogger('')
 547
 548         # get_current_container is invoked when we call
 549         # runtime_status_update, it is going to also raise an
 550         # exception.
 551         gcc_mock.side_effect = Exception("Second Error")
 552         try:
 553             root_logger.error("First Error")
 554         except Exception:
 555             self.fail("Exception in logger should not propagate")
 556         self.assertTrue(gcc_mock.called)
 557
 558     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 559     @mock.patch("arvados_cwl.util.get_current_container")
 560     @mock.patch("arvados.collection.CollectionReader")
 561     @mock.patch("arvados.collection.Collection")
 562     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 563         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 564
 565         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 566         self.assertTrue(gcc_mock.called)
 567
 568         runner.num_retries = 0
 569         runner.ignore_docker_for_reuse = False
 570         runner.intermediate_output_ttl = 0
 571         runner.secret_store = cwltool.secrets.SecretStore()
 572         runner.label = mock.MagicMock()
 573         runner.label.return_value = '[container testjob]'
 574
 575         runner.api.containers().get().execute.return_value = {
 576             "state":"Complete",
 577             "output": "abc+123",
 578             "exit_code": 1,
 579             "log": "def+234"
 580         }
 581
 582         col().open.return_value = []
 583
 584         loadingContext, runtimeContext = self.helper(runner)
 585
 586         arvjob = arvados_cwl.ArvadosContainer(runner,
 587                                               runtimeContext,
 588                                               mock.MagicMock(),
 589                                               {},
 590                                               None,
 591                                               [],
 592                                               [],
 593                                               "testjob")
 594         arvjob.output_callback = mock.MagicMock()
 595         arvjob.collect_outputs = mock.MagicMock()
 596         arvjob.successCodes = [0]
 597         arvjob.outdir = "/var/spool/cwl"
 598         arvjob.output_ttl = 3600
 599         arvjob.collect_outputs.return_value = {"out": "stuff"}
 600
 601         arvjob.done({
 602             "state": "Final",
 603             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 604             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 605             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 606             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 607             "modified_at": "2017-05-26T12:01:22Z"
 608         })
 609
 610         rts_mock.assert_called_with(
 611             'error',
 612             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 613             '  ** log is empty **'
 614         )
 615         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 616
 617     # The test passes no builder.resources
 618     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 619     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 620     def test_mounts(self, keepdocker):
 621         runner = mock.MagicMock()
 622         runner.ignore_docker_for_reuse = False
 623         runner.intermediate_output_ttl = 0
 624         runner.secret_store = cwltool.secrets.SecretStore()
 625
 626         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 627         runner.api.collections().get().execute.return_value = {
 628             "portable_data_hash": "99999999999999999999999999999994+99",
 629             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 630
 631         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 632
 633         tool = cmap({
 634             "inputs": [
 635                 {"id": "p1",
 636                  "type": "Directory"}
 637             ],
 638             "outputs": [],
 639             "baseCommand": "ls",
 640             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 641             "id": "#",
 642             "class": "CommandLineTool"
 643         })
 644
 645         loadingContext, runtimeContext = self.helper(runner)
 646         runtimeContext.name = "test_run_mounts"
 647
 648         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 649         arvtool.formatgraph = None
 650         job_order = {
 651             "p1": {
 652                 "class": "Directory",
 653                 "location": "keep:99999999999999999999999999999994+44",
 654                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 655                 "listing": [
 656                     {
 657                         "class": "File",
 658                         "location": "keep:99999999999999999999999999999994+44/file1",
 659                     },
 660                     {
 661                         "class": "File",
 662                         "location": "keep:99999999999999999999999999999994+44/file2",
 663                     }
 664                 ]
 665             }
 666         }
 667         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 668             j.run(runtimeContext)
 669             runner.api.container_requests().create.assert_called_with(
 670                 body=JsonDiffMatcher({
 671                     'environment': {
 672                         'HOME': '/var/spool/cwl',
 673                         'TMPDIR': '/tmp'
 674                     },
 675                     'name': 'test_run_mounts',
 676                     'runtime_constraints': {
 677                         'vcpus': 1,
 678                         'ram': 1073741824
 679                     },
 680                     'use_existing': True,
 681                     'priority': 500,
 682                     'mounts': {
 683                         "/keep/99999999999999999999999999999994+44": {
 684                             "kind": "collection",
 685                             "portable_data_hash": "99999999999999999999999999999994+44",
 686                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 687                         },
 688                         '/tmp': {'kind': 'tmp',
 689                                  "capacity": 1073741824 },
 690                         '/var/spool/cwl': {'kind': 'tmp',
 691                                            "capacity": 1073741824 }
 692                     },
 693                     'state': 'Committed',
 694                     'output_name': 'Output for step test_run_mounts',
 695                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 696                     'output_path': '/var/spool/cwl',
 697                     'output_ttl': 0,
 698                     'container_image': '99999999999999999999999999999994+99',
 699                     'command': ['ls', '/var/spool/cwl'],
 700                     'cwd': '/var/spool/cwl',
 701                     'scheduling_parameters': {},
 702                     'properties': {},
 703                     'secret_mounts': {},
 704                     'output_storage_classes': ["default"]
 705                 }))
 706
 707     # The test passes no builder.resources
 708     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 709     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 710     def test_secrets(self, keepdocker):
 711         runner = mock.MagicMock()
 712         runner.ignore_docker_for_reuse = False
 713         runner.intermediate_output_ttl = 0
 714         runner.secret_store = cwltool.secrets.SecretStore()
 715
 716         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 717         runner.api.collections().get().execute.return_value = {
 718             "portable_data_hash": "99999999999999999999999999999993+99"}
 719
 720         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 721
 722         tool = cmap({"arguments": ["md5sum", "example.conf"],
 723                      "class": "CommandLineTool",
 724                      "hints": [
 725                          {
 726                              "class": "http://commonwl.org/cwltool#Secrets",
 727                              "secrets": [
 728                                  "#secret_job.cwl/pw"
 729                              ]
 730                          }
 731                      ],
 732                      "id": "#secret_job.cwl",
 733                      "inputs": [
 734                          {
 735                              "id": "#secret_job.cwl/pw",
 736                              "type": "string"
 737                          }
 738                      ],
 739                      "outputs": [
 740                      ],
 741                      "requirements": [
 742                          {
 743                              "class": "InitialWorkDirRequirement",
 744                              "listing": [
 745                                  {
 746                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 747                                      "entryname": "example.conf"
 748                                  }
 749                              ]
 750                          }
 751                      ]})
 752
 753         loadingContext, runtimeContext = self.helper(runner)
 754         runtimeContext.name = "test_secrets"
 755
 756         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 757         arvtool.formatgraph = None
 758
 759         job_order = {"pw": "blorp"}
 760         runner.secret_store.store(["pw"], job_order)
 761
 762         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 763             j.run(runtimeContext)
 764             runner.api.container_requests().create.assert_called_with(
 765                 body=JsonDiffMatcher({
 766                     'environment': {
 767                         'HOME': '/var/spool/cwl',
 768                         'TMPDIR': '/tmp'
 769                     },
 770                     'name': 'test_secrets',
 771                     'runtime_constraints': {
 772                         'vcpus': 1,
 773                         'ram': 1073741824
 774                     },
 775                     'use_existing': True,
 776                     'priority': 500,
 777                     'mounts': {
 778                         '/tmp': {'kind': 'tmp',
 779                                  "capacity": 1073741824
 780                              },
 781                         '/var/spool/cwl': {'kind': 'tmp',
 782                                            "capacity": 1073741824 }
 783                     },
 784                     'state': 'Committed',
 785                     'output_name': 'Output for step test_secrets',
 786                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 787                     'output_path': '/var/spool/cwl',
 788                     'output_ttl': 0,
 789                     'container_image': '99999999999999999999999999999993+99',
 790                     'command': ['md5sum', 'example.conf'],
 791                     'cwd': '/var/spool/cwl',
 792                     'scheduling_parameters': {},
 793                     'properties': {},
 794                     "secret_mounts": {
 795                         "/var/spool/cwl/example.conf": {
 796                             "content": "username: user\npassword: blorp\n",
 797                             "kind": "text"
 798                         }
 799                     },
 800                     'output_storage_classes': ["default"]
 801                 }))
 802
 803     # The test passes no builder.resources
 804     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 805     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 806     def test_timelimit(self, keepdocker):
 807         runner = mock.MagicMock()
 808         runner.ignore_docker_for_reuse = False
 809         runner.intermediate_output_ttl = 0
 810         runner.secret_store = cwltool.secrets.SecretStore()
 811
 812         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 813         runner.api.collections().get().execute.return_value = {
 814             "portable_data_hash": "99999999999999999999999999999993+99"}
 815
 816         tool = cmap({
 817             "inputs": [],
 818             "outputs": [],
 819             "baseCommand": "ls",
 820             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 821             "id": "#",
 822             "class": "CommandLineTool",
 823             "hints": [
 824                 {
 825                     "class": "ToolTimeLimit",
 826                     "timelimit": 42
 827                 }
 828             ]
 829         })
 830
 831         loadingContext, runtimeContext = self.helper(runner)
 832         runtimeContext.name = "test_timelimit"
 833
 834         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 835         arvtool.formatgraph = None
 836
 837         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 838             j.run(runtimeContext)
 839
 840         _, kwargs = runner.api.container_requests().create.call_args
 841         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 842
 843
 844     # The test passes no builder.resources
 845     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 846     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 847     def test_setting_storage_class(self, keepdocker):
 848         arv_docker_clear_cache()
 849
 850         runner = mock.MagicMock()
 851         runner.ignore_docker_for_reuse = False
 852         runner.intermediate_output_ttl = 0
 853         runner.secret_store = cwltool.secrets.SecretStore()
 854
 855         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 856         runner.api.collections().get().execute.return_value = {
 857             "portable_data_hash": "99999999999999999999999999999993+99"}
 858
 859         tool = cmap({
 860             "inputs": [],
 861             "outputs": [],
 862             "baseCommand": "ls",
 863             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 864             "id": "#",
 865             "class": "CommandLineTool",
 866             "hints": [
 867                 {
 868                     "class": "http://arvados.org/cwl#OutputStorageClass",
 869                     "finalStorageClass": ["baz_sc", "qux_sc"],
 870                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 871                 }
 872             ]
 873         })
 874
 875         loadingContext, runtimeContext = self.helper(runner, True)
 876
 877         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 878         arvtool.formatgraph = None
 879
 880         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 881             j.run(runtimeContext)
 882             runner.api.container_requests().create.assert_called_with(
 883                 body=JsonDiffMatcher({
 884                     'environment': {
 885                         'HOME': '/var/spool/cwl',
 886                         'TMPDIR': '/tmp'
 887                     },
 888                     'name': 'test_run_True',
 889                     'runtime_constraints': {
 890                         'vcpus': 1,
 891                         'ram': 1073741824
 892                     },
 893                     'use_existing': True,
 894                     'priority': 500,
 895                     'mounts': {
 896                         '/tmp': {'kind': 'tmp',
 897                                  "capacity": 1073741824
 898                              },
 899                         '/var/spool/cwl': {'kind': 'tmp',
 900                                            "capacity": 1073741824 }
 901                     },
 902                     'state': 'Committed',
 903                     'output_name': 'Output for step test_run_True',
 904                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 905                     'output_path': '/var/spool/cwl',
 906                     'output_ttl': 0,
 907                     'container_image': '99999999999999999999999999999993+99',
 908                     'command': ['ls', '/var/spool/cwl'],
 909                     'cwd': '/var/spool/cwl',
 910                     'scheduling_parameters': {},
 911                     'properties': {},
 912                     'secret_mounts': {},
 913                     'output_storage_classes': ["foo_sc", "bar_sc"]
 914                 }))
 915
 916
 917 class TestWorkflow(unittest.TestCase):
 918     def setUp(self):
 919         cwltool.process._names = set()
 920         arv_docker_clear_cache()
 921
 922     def helper(self, runner, enable_reuse=True):
 923         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 924
 925         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 926                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 927
 928         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 929         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 930         document_loader.fetch_text = document_loader.fetcher.fetch_text
 931         document_loader.check_exists = document_loader.fetcher.check_exists
 932
 933         loadingContext = arvados_cwl.context.ArvLoadingContext(
 934             {"avsc_names": avsc_names,
 935              "basedir": "",
 936              "make_fs_access": make_fs_access,
 937              "loader": document_loader,
 938              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 939              "construct_tool_object": runner.arv_make_tool})
 940         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 941             {"work_api": "containers",
 942              "basedir": "",
 943              "name": "test_run_wf_"+str(enable_reuse),
 944              "make_fs_access": make_fs_access,
 945              "tmpdir": "/tmp",
 946              "enable_reuse": enable_reuse,
 947              "priority": 500})
 948
 949         return loadingContext, runtimeContext
 950
 951     # The test passes no builder.resources
 952     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 953     @mock.patch("arvados.collection.CollectionReader")
 954     @mock.patch("arvados.collection.Collection")
 955     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 956     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 957         arvados_cwl.add_arv_hints()
 958
 959         api = mock.MagicMock()
 960         api._rootDesc = get_rootDesc()
 961
 962         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 963         self.assertEqual(runner.work_api, 'containers')
 964
 965         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 966         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 967         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 968                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 969
 970         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 971         runner.ignore_docker_for_reuse = False
 972         runner.num_retries = 0
 973         runner.secret_store = cwltool.secrets.SecretStore()
 974
 975         loadingContext, runtimeContext = self.helper(runner)
 976         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 977
 978         mockcollectionreader().exists.return_value = True
 979
 980         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 981         metadata["cwlVersion"] = tool["cwlVersion"]
 982
 983         mockc = mock.MagicMock()
 984         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 985         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 986
 987         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 988         arvtool.formatgraph = None
 989         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 990
 991         next(it).run(runtimeContext)
 992         next(it).run(runtimeContext)
 993
 994         with open("tests/wf/scatter2_subwf.cwl") as f:
 995             subwf = StripYAMLComments(f.read()).rstrip()
 996
 997         runner.api.container_requests().create.assert_called_with(
 998             body=JsonDiffMatcher({
 999                 "command": [
1000                     "cwltool",
1001                     "--no-container",
1002                     "--move-outputs",
1003                     "--preserve-entire-environment",
1004                     "workflow.cwl",
1005                     "cwl.input.yml"
1006                 ],
1007                 "container_image": "99999999999999999999999999999993+99",
1008                 "cwd": "/var/spool/cwl",
1009                 "environment": {
1010                     "HOME": "/var/spool/cwl",
1011                     "TMPDIR": "/tmp"
1012                 },
1013                 "mounts": {
1014                     "/keep/99999999999999999999999999999999+118": {
1015                         "kind": "collection",
1016                         "portable_data_hash": "99999999999999999999999999999999+118"
1017                     },
1018                     "/tmp": {
1019                         "capacity": 1073741824,
1020                         "kind": "tmp"
1021                     },
1022                     "/var/spool/cwl": {
1023                         "capacity": 1073741824,
1024                         "kind": "tmp"
1025                     },
1026                     "/var/spool/cwl/cwl.input.yml": {
1027                         "kind": "collection",
1028                         "path": "cwl.input.yml",
1029                         "portable_data_hash": "99999999999999999999999999999996+99"
1030                     },
1031                     "/var/spool/cwl/workflow.cwl": {
1032                         "kind": "collection",
1033                         "path": "workflow.cwl",
1034                         "portable_data_hash": "99999999999999999999999999999996+99"
1035                     },
1036                     "stdout": {
1037                         "kind": "file",
1038                         "path": "/var/spool/cwl/cwl.output.json"
1039                     }
1040                 },
1041                 "name": "scatterstep",
1042                 "output_name": "Output for step scatterstep",
1043                 "output_path": "/var/spool/cwl",
1044                 "output_ttl": 0,
1045                 "priority": 500,
1046                 "properties": {},
1047                 "runtime_constraints": {
1048                     "ram": 1073741824,
1049                     "vcpus": 1
1050                 },
1051                 "scheduling_parameters": {},
1052                 "secret_mounts": {},
1053                 "state": "Committed",
1054                 "use_existing": True,
1055                 'output_storage_classes': ["default"]
1056             }))
1057         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1058         mockc.open().__enter__().write.assert_has_calls([mock.call(
1059 '''{
1060   "fileblub": {
1061     "basename": "token.txt",
1062     "class": "File",
1063     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1064     "size": 0
1065   },
1066   "sleeptime": 5
1067 }''')])
1068
1069     # The test passes no builder.resources
1070     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1071     @mock.patch("arvados.collection.CollectionReader")
1072     @mock.patch("arvados.collection.Collection")
1073     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1074     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1075         arvados_cwl.add_arv_hints()
1076
1077         api = mock.MagicMock()
1078         api._rootDesc = get_rootDesc()
1079
1080         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1081         self.assertEqual(runner.work_api, 'containers')
1082
1083         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1084         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1085                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1086         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1087                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1088
1089         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1090         runner.ignore_docker_for_reuse = False
1091         runner.num_retries = 0
1092         runner.secret_store = cwltool.secrets.SecretStore()
1093
1094         loadingContext, runtimeContext = self.helper(runner)
1095         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1096         loadingContext.do_update = True
1097         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1098
1099         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1100
1101         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1102         arvtool.formatgraph = None
1103         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1104
1105         next(it).run(runtimeContext)
1106         next(it).run(runtimeContext)
1107
1108         with open("tests/wf/echo-subwf.cwl") as f:
1109             subwf = StripYAMLComments(f.read())
1110
1111         runner.api.container_requests().create.assert_called_with(
1112             body=JsonDiffMatcher({
1113                 'output_ttl': 0,
1114                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1115                 'scheduling_parameters': {},
1116                 'name': u'echo-subwf',
1117                 'secret_mounts': {},
1118                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1119                 'properties': {},
1120                 'priority': 500,
1121                 'mounts': {
1122                     '/var/spool/cwl/cwl.input.yml': {
1123                         'portable_data_hash': '99999999999999999999999999999996+99',
1124                         'kind': 'collection',
1125                         'path': 'cwl.input.yml'
1126                     },
1127                     '/var/spool/cwl/workflow.cwl': {
1128                         'portable_data_hash': '99999999999999999999999999999996+99',
1129                         'kind': 'collection',
1130                         'path': 'workflow.cwl'
1131                     },
1132                     'stdout': {
1133                         'path': '/var/spool/cwl/cwl.output.json',
1134                         'kind': 'file'
1135                     },
1136                     '/tmp': {
1137                         'kind': 'tmp',
1138                         'capacity': 1073741824
1139                     }, '/var/spool/cwl': {
1140                         'kind': 'tmp',
1141                         'capacity': 3221225472
1142                     }
1143                 },
1144                 'state': 'Committed',
1145                 'output_path': '/var/spool/cwl',
1146                 'container_image': '99999999999999999999999999999993+99',
1147                 'command': [
1148                     u'cwltool',
1149                     u'--no-container',
1150                     u'--move-outputs',
1151                     u'--preserve-entire-environment',
1152                     u'workflow.cwl',
1153                     u'cwl.input.yml'
1154                 ],
1155                 'use_existing': True,
1156                 'output_name': u'Output for step echo-subwf',
1157                 'cwd': '/var/spool/cwl',
1158                 'output_storage_classes': ["default"]
1159             }))
1160
1161     def test_default_work_api(self):
1162         arvados_cwl.add_arv_hints()
1163
1164         api = mock.MagicMock()
1165         api._rootDesc = copy.deepcopy(get_rootDesc())
1166         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1167         self.assertEqual(runner.work_api, 'containers')