sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from cwltool.update import INTERNAL_VERSION
  22 from schema_salad.ref_resolver import Loader
  23 from schema_salad.sourceline import cmap
  24
  25 from .matcher import JsonDiffMatcher, StripYAMLComments
  26 from .mock_discovery import get_rootDesc
  27
  28 if not os.getenv('ARVADOS_DEBUG'):
  29     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  30     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  31
  32 class CollectionMock(object):
  33     def __init__(self, vwdmock, *args, **kwargs):
  34         self.vwdmock = vwdmock
  35         self.count = 0
  36
  37     def open(self, *args, **kwargs):
  38         self.count += 1
  39         return self.vwdmock.open(*args, **kwargs)
  40
  41     def copy(self, *args, **kwargs):
  42         self.count += 1
  43         self.vwdmock.copy(*args, **kwargs)
  44
  45     def save_new(self, *args, **kwargs):
  46         pass
  47
  48     def __len__(self):
  49         return self.count
  50
  51     def portable_data_hash(self):
  52         if self.count == 0:
  53             return arvados.config.EMPTY_BLOCK_LOCATOR
  54         else:
  55             return "99999999999999999999999999999996+99"
  56
  57
  58 class TestContainer(unittest.TestCase):
  59
  60     def setUp(self):
  61         cwltool.process._names = set()
  62         arv_docker_clear_cache()
  63
  64     def helper(self, runner, enable_reuse=True):
  65         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  66
  67         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  68                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  69         loadingContext = arvados_cwl.context.ArvLoadingContext(
  70             {"avsc_names": avsc_names,
  71              "basedir": "",
  72              "make_fs_access": make_fs_access,
  73              "loader": Loader({}),
  74              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  75         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  76             {"work_api": "containers",
  77              "basedir": "",
  78              "name": "test_run_"+str(enable_reuse),
  79              "make_fs_access": make_fs_access,
  80              "tmpdir": "/tmp",
  81              "enable_reuse": enable_reuse,
  82              "priority": 500,
  83              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84             })
  85
  86         return loadingContext, runtimeContext
  87
  88     # Helper function to set up the ArvCwlExecutor to use the containers api
  89     # and test that the RuntimeStatusLoggingHandler is set up correctly
  90     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  91         api = mock.MagicMock()
  92         api._rootDesc = copy.deepcopy(get_rootDesc())
  93
  94         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  95         # adds the logging handler that will call runtime_status_update() mock
  96         self.assertFalse(gcc_mock.called)
  97         runner = arvados_cwl.ArvCwlExecutor(api)
  98         self.assertEqual(runner.work_api, 'containers')
  99         root_logger = logging.getLogger('')
 100         handlerClasses = [h.__class__ for h in root_logger.handlers]
 101         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 102         return runner
 103
 104     # The test passes no builder.resources
 105     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 106     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 107     def test_run(self, keepdocker):
 108         for enable_reuse in (True, False):
 109             arv_docker_clear_cache()
 110
 111             runner = mock.MagicMock()
 112             runner.ignore_docker_for_reuse = False
 113             runner.intermediate_output_ttl = 0
 114             runner.secret_store = cwltool.secrets.SecretStore()
 115             runner.api._rootDesc = {"revision": "20210628"}
 116
 117             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 118             runner.api.collections().get().execute.return_value = {
 119                 "portable_data_hash": "99999999999999999999999999999993+99"}
 120
 121             tool = cmap({
 122                 "inputs": [],
 123                 "outputs": [],
 124                 "baseCommand": "ls",
 125                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 126                 "id": "#",
 127                 "class": "org.w3id.cwl.cwl.CommandLineTool"
 128             })
 129
 130             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 131
 132             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 133             arvtool.formatgraph = None
 134
 135             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 136                 j.run(runtimeContext)
 137                 runner.api.container_requests().create.assert_called_with(
 138                     body=JsonDiffMatcher({
 139                         'environment': {
 140                             'HOME': '/var/spool/cwl',
 141                             'TMPDIR': '/tmp'
 142                         },
 143                         'name': 'test_run_'+str(enable_reuse),
 144                         'runtime_constraints': {
 145                             'vcpus': 1,
 146                             'ram': 1073741824
 147                         },
 148                         'use_existing': enable_reuse,
 149                         'priority': 500,
 150                         'mounts': {
 151                             '/tmp': {'kind': 'tmp',
 152                                      "capacity": 1073741824
 153                                  },
 154                             '/var/spool/cwl': {'kind': 'tmp',
 155                                                "capacity": 1073741824 }
 156                         },
 157                         'state': 'Committed',
 158                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 159                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 160                         'output_path': '/var/spool/cwl',
 161                         'output_ttl': 0,
 162                         'container_image': '99999999999999999999999999999993+99',
 163                         'command': ['ls', '/var/spool/cwl'],
 164                         'cwd': '/var/spool/cwl',
 165                         'scheduling_parameters': {},
 166                         'properties': {},
 167                         'secret_mounts': {},
 168                         'output_storage_classes': ["default"]
 169                     }))
 170
 171     # The test passes some fields in builder.resources
 172     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 173     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 174     def test_resource_requirements(self, keepdocker):
 175         runner = mock.MagicMock()
 176         runner.ignore_docker_for_reuse = False
 177         runner.intermediate_output_ttl = 3600
 178         runner.secret_store = cwltool.secrets.SecretStore()
 179         runner.api._rootDesc = {"revision": "20210628"}
 180
 181         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 182         runner.api.collections().get().execute.return_value = {
 183             "portable_data_hash": "99999999999999999999999999999993+99"}
 184
 185         tool = cmap({
 186             "inputs": [],
 187             "outputs": [],
 188             "hints": [{
 189                 "class": "ResourceRequirement",
 190                 "coresMin": 3,
 191                 "ramMin": 3000,
 192                 "tmpdirMin": 4000,
 193                 "outdirMin": 5000
 194             }, {
 195                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 196                 "keep_cache": 512
 197             }, {
 198                 "class": "http://arvados.org/cwl#APIRequirement",
 199             }, {
 200                 "class": "http://arvados.org/cwl#PartitionRequirement",
 201                 "partition": "blurb"
 202             }, {
 203                 "class": "http://arvados.org/cwl#IntermediateOutput",
 204                 "outputTTL": 7200
 205             }, {
 206                 "class": "http://arvados.org/cwl#ReuseRequirement",
 207                 "enableReuse": False
 208             }],
 209             "baseCommand": "ls",
 210             "id": "#",
 211             "class": "org.w3id.cwl.cwl.CommandLineTool"
 212         })
 213
 214         loadingContext, runtimeContext = self.helper(runner)
 215         runtimeContext.name = "test_resource_requirements"
 216
 217         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 218         arvtool.formatgraph = None
 219         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 220             j.run(runtimeContext)
 221
 222         call_args, call_kwargs = runner.api.container_requests().create.call_args
 223
 224         call_body_expected = {
 225             'environment': {
 226                 'HOME': '/var/spool/cwl',
 227                 'TMPDIR': '/tmp'
 228             },
 229             'name': 'test_resource_requirements',
 230             'runtime_constraints': {
 231                 'vcpus': 3,
 232                 'ram': 3145728000,
 233                 'keep_cache_ram': 536870912,
 234                 'API': True
 235             },
 236             'use_existing': False,
 237             'priority': 500,
 238             'mounts': {
 239                 '/tmp': {'kind': 'tmp',
 240                          "capacity": 4194304000 },
 241                 '/var/spool/cwl': {'kind': 'tmp',
 242                                    "capacity": 5242880000 }
 243             },
 244             'state': 'Committed',
 245             'output_name': 'Output for step test_resource_requirements',
 246             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 247             'output_path': '/var/spool/cwl',
 248             'output_ttl': 7200,
 249             'container_image': '99999999999999999999999999999993+99',
 250             'command': ['ls'],
 251             'cwd': '/var/spool/cwl',
 252             'scheduling_parameters': {
 253                 'partitions': ['blurb']
 254             },
 255             'properties': {},
 256             'secret_mounts': {},
 257             'output_storage_classes': ["default"]
 258         }
 259
 260         call_body = call_kwargs.get('body', None)
 261         self.assertNotEqual(None, call_body)
 262         for key in call_body:
 263             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 264
 265
 266     # The test passes some fields in builder.resources
 267     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 268     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 269     @mock.patch("arvados.collection.Collection")
 270     def test_initial_work_dir(self, collection_mock, keepdocker):
 271         runner = mock.MagicMock()
 272         runner.ignore_docker_for_reuse = False
 273         runner.intermediate_output_ttl = 0
 274         runner.secret_store = cwltool.secrets.SecretStore()
 275         runner.api._rootDesc = {"revision": "20210628"}
 276
 277         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 278         runner.api.collections().get().execute.return_value = {
 279             "portable_data_hash": "99999999999999999999999999999993+99"}
 280
 281         sourcemock = mock.MagicMock()
 282         def get_collection_mock(p):
 283             if "/" in p:
 284                 return (sourcemock, p.split("/", 1)[1])
 285             else:
 286                 return (sourcemock, "")
 287         runner.fs_access.get_collection.side_effect = get_collection_mock
 288
 289         vwdmock = mock.MagicMock()
 290         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 291
 292         tool = cmap({
 293             "inputs": [],
 294             "outputs": [],
 295             "hints": [{
 296                 "class": "InitialWorkDirRequirement",
 297                 "listing": [{
 298                     "class": "File",
 299                     "basename": "foo",
 300                     "location": "keep:99999999999999999999999999999995+99/bar"
 301                 },
 302                 {
 303                     "class": "Directory",
 304                     "basename": "foo2",
 305                     "location": "keep:99999999999999999999999999999995+99"
 306                 },
 307                 {
 308                     "class": "File",
 309                     "basename": "filename",
 310                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 311                 },
 312                 {
 313                     "class": "Directory",
 314                     "basename": "subdir",
 315                     "location": "keep:99999999999999999999999999999995+99/subdir"
 316                 }                        ]
 317             }],
 318             "baseCommand": "ls",
 319             "id": "#",
 320             "class": "org.w3id.cwl.cwl.CommandLineTool"
 321         })
 322
 323         loadingContext, runtimeContext = self.helper(runner)
 324         runtimeContext.name = "test_initial_work_dir"
 325
 326         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 327         arvtool.formatgraph = None
 328         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 329             j.run(runtimeContext)
 330
 331         call_args, call_kwargs = runner.api.container_requests().create.call_args
 332
 333         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 334         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 335         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 336         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 337
 338         call_body_expected = {
 339             'environment': {
 340                 'HOME': '/var/spool/cwl',
 341                 'TMPDIR': '/tmp'
 342             },
 343             'name': 'test_initial_work_dir',
 344             'runtime_constraints': {
 345                 'vcpus': 1,
 346                 'ram': 1073741824
 347             },
 348             'use_existing': True,
 349             'priority': 500,
 350             'mounts': {
 351                 '/tmp': {'kind': 'tmp',
 352                          "capacity": 1073741824 },
 353                 '/var/spool/cwl': {'kind': 'tmp',
 354                                    "capacity": 1073741824 },
 355                 '/var/spool/cwl/foo': {
 356                     'kind': 'collection',
 357                     'path': 'foo',
 358                     'portable_data_hash': '99999999999999999999999999999996+99'
 359                 },
 360                 '/var/spool/cwl/foo2': {
 361                     'kind': 'collection',
 362                     'path': 'foo2',
 363                     'portable_data_hash': '99999999999999999999999999999996+99'
 364                 },
 365                 '/var/spool/cwl/filename': {
 366                     'kind': 'collection',
 367                     'path': 'filename',
 368                     'portable_data_hash': '99999999999999999999999999999996+99'
 369                 },
 370                 '/var/spool/cwl/subdir': {
 371                     'kind': 'collection',
 372                     'path': 'subdir',
 373                     'portable_data_hash': '99999999999999999999999999999996+99'
 374                 }
 375             },
 376             'state': 'Committed',
 377             'output_name': 'Output for step test_initial_work_dir',
 378             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 379             'output_path': '/var/spool/cwl',
 380             'output_ttl': 0,
 381             'container_image': '99999999999999999999999999999993+99',
 382             'command': ['ls'],
 383             'cwd': '/var/spool/cwl',
 384             'scheduling_parameters': {
 385             },
 386             'properties': {},
 387             'secret_mounts': {},
 388             'output_storage_classes': ["default"]
 389         }
 390
 391         call_body = call_kwargs.get('body', None)
 392         self.assertNotEqual(None, call_body)
 393         for key in call_body:
 394             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 395
 396
 397     # Test redirecting stdin/stdout/stderr
 398     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 399     def test_redirects(self, keepdocker):
 400         runner = mock.MagicMock()
 401         runner.ignore_docker_for_reuse = False
 402         runner.intermediate_output_ttl = 0
 403         runner.secret_store = cwltool.secrets.SecretStore()
 404         runner.api._rootDesc = {"revision": "20210628"}
 405
 406         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 407         runner.api.collections().get().execute.return_value = {
 408             "portable_data_hash": "99999999999999999999999999999993+99"}
 409
 410         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 411
 412         tool = cmap({
 413             "inputs": [],
 414             "outputs": [],
 415             "baseCommand": "ls",
 416             "stdout": "stdout.txt",
 417             "stderr": "stderr.txt",
 418             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 419             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 420             "id": "#",
 421             "class": "org.w3id.cwl.cwl.CommandLineTool"
 422         })
 423
 424         loadingContext, runtimeContext = self.helper(runner)
 425         runtimeContext.name = "test_run_redirect"
 426
 427         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 428         arvtool.formatgraph = None
 429         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 430             j.run(runtimeContext)
 431             runner.api.container_requests().create.assert_called_with(
 432                 body=JsonDiffMatcher({
 433                     'environment': {
 434                         'HOME': '/var/spool/cwl',
 435                         'TMPDIR': '/tmp'
 436                     },
 437                     'name': 'test_run_redirect',
 438                     'runtime_constraints': {
 439                         'vcpus': 1,
 440                         'ram': 1073741824
 441                     },
 442                     'use_existing': True,
 443                     'priority': 500,
 444                     'mounts': {
 445                         '/tmp': {'kind': 'tmp',
 446                                  "capacity": 1073741824 },
 447                         '/var/spool/cwl': {'kind': 'tmp',
 448                                            "capacity": 1073741824 },
 449                         "stderr": {
 450                             "kind": "file",
 451                             "path": "/var/spool/cwl/stderr.txt"
 452                         },
 453                         "stdin": {
 454                             "kind": "collection",
 455                             "path": "file.txt",
 456                             "portable_data_hash": "99999999999999999999999999999996+99"
 457                         },
 458                         "stdout": {
 459                             "kind": "file",
 460                             "path": "/var/spool/cwl/stdout.txt"
 461                         },
 462                     },
 463                     'state': 'Committed',
 464                     "output_name": "Output for step test_run_redirect",
 465                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 466                     'output_path': '/var/spool/cwl',
 467                     'output_ttl': 0,
 468                     'container_image': '99999999999999999999999999999993+99',
 469                     'command': ['ls', '/var/spool/cwl'],
 470                     'cwd': '/var/spool/cwl',
 471                     'scheduling_parameters': {},
 472                     'properties': {},
 473                     'secret_mounts': {},
 474                     'output_storage_classes': ["default"]
 475                 }))
 476
 477     @mock.patch("arvados.collection.Collection")
 478     def test_done(self, col):
 479         api = mock.MagicMock()
 480
 481         runner = mock.MagicMock()
 482         runner.api = api
 483         runner.num_retries = 0
 484         runner.ignore_docker_for_reuse = False
 485         runner.intermediate_output_ttl = 0
 486         runner.secret_store = cwltool.secrets.SecretStore()
 487
 488         runner.api.containers().get().execute.return_value = {"state":"Complete",
 489                                                               "output": "abc+123",
 490                                                               "exit_code": 0}
 491
 492         col().open.return_value = []
 493
 494         loadingContext, runtimeContext = self.helper(runner)
 495
 496         arvjob = arvados_cwl.ArvadosContainer(runner,
 497                                               runtimeContext,
 498                                               mock.MagicMock(),
 499                                               {},
 500                                               None,
 501                                               [],
 502                                               [],
 503                                               "testjob")
 504         arvjob.output_callback = mock.MagicMock()
 505         arvjob.collect_outputs = mock.MagicMock()
 506         arvjob.successCodes = [0]
 507         arvjob.outdir = "/var/spool/cwl"
 508         arvjob.output_ttl = 3600
 509
 510         arvjob.collect_outputs.return_value = {"out": "stuff"}
 511
 512         arvjob.done({
 513             "state": "Final",
 514             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 515             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 516             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 517             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 518             "modified_at": "2017-05-26T12:01:22Z"
 519         })
 520
 521         self.assertFalse(api.collections().create.called)
 522         self.assertFalse(runner.runtime_status_error.called)
 523
 524         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 525         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 526         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 527
 528     # Test to make sure we dont call runtime_status_update if we already did
 529     # some where higher up in the call stack
 530     @mock.patch("arvados_cwl.util.get_current_container")
 531     def test_recursive_runtime_status_update(self, gcc_mock):
 532         self.setup_and_test_container_executor_and_logging(gcc_mock)
 533         root_logger = logging.getLogger('')
 534
 535         # get_current_container is invoked when we call runtime_status_update
 536         # so try and log again!
 537         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 538         try:
 539             root_logger.error("First Error")
 540         except RuntimeError:
 541             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 542
 543
 544     # Test to make sure that an exception raised from
 545     # get_current_container doesn't cause the logger to raise an
 546     # exception
 547     @mock.patch("arvados_cwl.util.get_current_container")
 548     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 549         self.setup_and_test_container_executor_and_logging(gcc_mock)
 550         root_logger = logging.getLogger('')
 551
 552         # get_current_container is invoked when we call
 553         # runtime_status_update, it is going to also raise an
 554         # exception.
 555         gcc_mock.side_effect = Exception("Second Error")
 556         try:
 557             root_logger.error("First Error")
 558         except Exception:
 559             self.fail("Exception in logger should not propagate")
 560         self.assertTrue(gcc_mock.called)
 561
 562     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 563     @mock.patch("arvados_cwl.util.get_current_container")
 564     @mock.patch("arvados.collection.CollectionReader")
 565     @mock.patch("arvados.collection.Collection")
 566     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 567         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 568
 569         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 570         self.assertTrue(gcc_mock.called)
 571
 572         runner.num_retries = 0
 573         runner.ignore_docker_for_reuse = False
 574         runner.intermediate_output_ttl = 0
 575         runner.secret_store = cwltool.secrets.SecretStore()
 576         runner.label = mock.MagicMock()
 577         runner.label.return_value = '[container testjob]'
 578
 579         runner.api.containers().get().execute.return_value = {
 580             "state":"Complete",
 581             "output": "abc+123",
 582             "exit_code": 1,
 583             "log": "def+234"
 584         }
 585
 586         col().open.return_value = []
 587
 588         loadingContext, runtimeContext = self.helper(runner)
 589
 590         arvjob = arvados_cwl.ArvadosContainer(runner,
 591                                               runtimeContext,
 592                                               mock.MagicMock(),
 593                                               {},
 594                                               None,
 595                                               [],
 596                                               [],
 597                                               "testjob")
 598         arvjob.output_callback = mock.MagicMock()
 599         arvjob.collect_outputs = mock.MagicMock()
 600         arvjob.successCodes = [0]
 601         arvjob.outdir = "/var/spool/cwl"
 602         arvjob.output_ttl = 3600
 603         arvjob.collect_outputs.return_value = {"out": "stuff"}
 604
 605         arvjob.done({
 606             "state": "Final",
 607             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 608             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 609             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 610             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 611             "modified_at": "2017-05-26T12:01:22Z"
 612         })
 613
 614         rts_mock.assert_called_with(
 615             'error',
 616             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 617             '  ** log is empty **'
 618         )
 619         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 620
 621     # The test passes no builder.resources
 622     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 623     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 624     def test_mounts(self, keepdocker):
 625         runner = mock.MagicMock()
 626         runner.ignore_docker_for_reuse = False
 627         runner.intermediate_output_ttl = 0
 628         runner.secret_store = cwltool.secrets.SecretStore()
 629         runner.api._rootDesc = {"revision": "20210628"}
 630
 631         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 632         runner.api.collections().get().execute.return_value = {
 633             "portable_data_hash": "99999999999999999999999999999994+99",
 634             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 635
 636         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 637
 638         tool = cmap({
 639             "inputs": [
 640                 {"id": "p1",
 641                  "type": "Directory"}
 642             ],
 643             "outputs": [],
 644             "baseCommand": "ls",
 645             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 646             "id": "#",
 647             "class": "org.w3id.cwl.cwl.CommandLineTool"
 648         })
 649
 650         loadingContext, runtimeContext = self.helper(runner)
 651         runtimeContext.name = "test_run_mounts"
 652
 653         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 654         arvtool.formatgraph = None
 655         job_order = {
 656             "p1": {
 657                 "class": "Directory",
 658                 "location": "keep:99999999999999999999999999999994+44",
 659                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 660                 "listing": [
 661                     {
 662                         "class": "File",
 663                         "location": "keep:99999999999999999999999999999994+44/file1",
 664                     },
 665                     {
 666                         "class": "File",
 667                         "location": "keep:99999999999999999999999999999994+44/file2",
 668                     }
 669                 ]
 670             }
 671         }
 672         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 673             j.run(runtimeContext)
 674             runner.api.container_requests().create.assert_called_with(
 675                 body=JsonDiffMatcher({
 676                     'environment': {
 677                         'HOME': '/var/spool/cwl',
 678                         'TMPDIR': '/tmp'
 679                     },
 680                     'name': 'test_run_mounts',
 681                     'runtime_constraints': {
 682                         'vcpus': 1,
 683                         'ram': 1073741824
 684                     },
 685                     'use_existing': True,
 686                     'priority': 500,
 687                     'mounts': {
 688                         "/keep/99999999999999999999999999999994+44": {
 689                             "kind": "collection",
 690                             "portable_data_hash": "99999999999999999999999999999994+44",
 691                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 692                         },
 693                         '/tmp': {'kind': 'tmp',
 694                                  "capacity": 1073741824 },
 695                         '/var/spool/cwl': {'kind': 'tmp',
 696                                            "capacity": 1073741824 }
 697                     },
 698                     'state': 'Committed',
 699                     'output_name': 'Output for step test_run_mounts',
 700                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 701                     'output_path': '/var/spool/cwl',
 702                     'output_ttl': 0,
 703                     'container_image': '99999999999999999999999999999994+99',
 704                     'command': ['ls', '/var/spool/cwl'],
 705                     'cwd': '/var/spool/cwl',
 706                     'scheduling_parameters': {},
 707                     'properties': {},
 708                     'secret_mounts': {},
 709                     'output_storage_classes': ["default"]
 710                 }))
 711
 712     # The test passes no builder.resources
 713     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 714     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 715     def test_secrets(self, keepdocker):
 716         runner = mock.MagicMock()
 717         runner.ignore_docker_for_reuse = False
 718         runner.intermediate_output_ttl = 0
 719         runner.secret_store = cwltool.secrets.SecretStore()
 720         runner.api._rootDesc = {"revision": "20210628"}
 721
 722         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 723         runner.api.collections().get().execute.return_value = {
 724             "portable_data_hash": "99999999999999999999999999999993+99"}
 725
 726         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 727
 728         tool = cmap({"arguments": ["md5sum", "example.conf"],
 729                      "class": "org.w3id.cwl.cwl.CommandLineTool",
 730                      "hints": [
 731                          {
 732                              "class": "http://commonwl.org/cwltool#Secrets",
 733                              "secrets": [
 734                                  "#secret_job.cwl/pw"
 735                              ]
 736                          }
 737                      ],
 738                      "id": "#secret_job.cwl",
 739                      "inputs": [
 740                          {
 741                              "id": "#secret_job.cwl/pw",
 742                              "type": "string"
 743                          }
 744                      ],
 745                      "outputs": [
 746                      ],
 747                      "requirements": [
 748                          {
 749                              "class": "InitialWorkDirRequirement",
 750                              "listing": [
 751                                  {
 752                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 753                                      "entryname": "example.conf"
 754                                  }
 755                              ]
 756                          }
 757                      ]})
 758
 759         loadingContext, runtimeContext = self.helper(runner)
 760         runtimeContext.name = "test_secrets"
 761
 762         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 763         arvtool.formatgraph = None
 764
 765         job_order = {"pw": "blorp"}
 766         runner.secret_store.store(["pw"], job_order)
 767
 768         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 769             j.run(runtimeContext)
 770             runner.api.container_requests().create.assert_called_with(
 771                 body=JsonDiffMatcher({
 772                     'environment': {
 773                         'HOME': '/var/spool/cwl',
 774                         'TMPDIR': '/tmp'
 775                     },
 776                     'name': 'test_secrets',
 777                     'runtime_constraints': {
 778                         'vcpus': 1,
 779                         'ram': 1073741824
 780                     },
 781                     'use_existing': True,
 782                     'priority': 500,
 783                     'mounts': {
 784                         '/tmp': {'kind': 'tmp',
 785                                  "capacity": 1073741824
 786                              },
 787                         '/var/spool/cwl': {'kind': 'tmp',
 788                                            "capacity": 1073741824 }
 789                     },
 790                     'state': 'Committed',
 791                     'output_name': 'Output for step test_secrets',
 792                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 793                     'output_path': '/var/spool/cwl',
 794                     'output_ttl': 0,
 795                     'container_image': '99999999999999999999999999999993+99',
 796                     'command': ['md5sum', 'example.conf'],
 797                     'cwd': '/var/spool/cwl',
 798                     'scheduling_parameters': {},
 799                     'properties': {},
 800                     "secret_mounts": {
 801                         "/var/spool/cwl/example.conf": {
 802                             "content": "username: user\npassword: blorp\n",
 803                             "kind": "text"
 804                         }
 805                     },
 806                     'output_storage_classes': ["default"]
 807                 }))
 808
 809     # The test passes no builder.resources
 810     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 811     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 812     def test_timelimit(self, keepdocker):
 813         runner = mock.MagicMock()
 814         runner.ignore_docker_for_reuse = False
 815         runner.intermediate_output_ttl = 0
 816         runner.secret_store = cwltool.secrets.SecretStore()
 817         runner.api._rootDesc = {"revision": "20210628"}
 818
 819         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 820         runner.api.collections().get().execute.return_value = {
 821             "portable_data_hash": "99999999999999999999999999999993+99"}
 822
 823         tool = cmap({
 824             "inputs": [],
 825             "outputs": [],
 826             "baseCommand": "ls",
 827             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 828             "id": "#",
 829             "class": "org.w3id.cwl.cwl.CommandLineTool",
 830             "hints": [
 831                 {
 832                     "class": "ToolTimeLimit",
 833                     "timelimit": 42
 834                 }
 835             ]
 836         })
 837
 838         loadingContext, runtimeContext = self.helper(runner)
 839         runtimeContext.name = "test_timelimit"
 840
 841         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 842         arvtool.formatgraph = None
 843
 844         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 845             j.run(runtimeContext)
 846
 847         _, kwargs = runner.api.container_requests().create.call_args
 848         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 849
 850
 851     # The test passes no builder.resources
 852     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 853     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 854     def test_setting_storage_class(self, keepdocker):
 855         arv_docker_clear_cache()
 856
 857         runner = mock.MagicMock()
 858         runner.ignore_docker_for_reuse = False
 859         runner.intermediate_output_ttl = 0
 860         runner.secret_store = cwltool.secrets.SecretStore()
 861         runner.api._rootDesc = {"revision": "20210628"}
 862
 863         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 864         runner.api.collections().get().execute.return_value = {
 865             "portable_data_hash": "99999999999999999999999999999993+99"}
 866
 867         tool = cmap({
 868             "inputs": [],
 869             "outputs": [],
 870             "baseCommand": "ls",
 871             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 872             "id": "#",
 873             "class": "org.w3id.cwl.cwl.CommandLineTool",
 874             "hints": [
 875                 {
 876                     "class": "http://arvados.org/cwl#OutputStorageClass",
 877                     "finalStorageClass": ["baz_sc", "qux_sc"],
 878                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 879                 }
 880             ]
 881         })
 882
 883         loadingContext, runtimeContext = self.helper(runner, True)
 884
 885         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 886         arvtool.formatgraph = None
 887
 888         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 889             j.run(runtimeContext)
 890             runner.api.container_requests().create.assert_called_with(
 891                 body=JsonDiffMatcher({
 892                     'environment': {
 893                         'HOME': '/var/spool/cwl',
 894                         'TMPDIR': '/tmp'
 895                     },
 896                     'name': 'test_run_True',
 897                     'runtime_constraints': {
 898                         'vcpus': 1,
 899                         'ram': 1073741824
 900                     },
 901                     'use_existing': True,
 902                     'priority': 500,
 903                     'mounts': {
 904                         '/tmp': {'kind': 'tmp',
 905                                  "capacity": 1073741824
 906                              },
 907                         '/var/spool/cwl': {'kind': 'tmp',
 908                                            "capacity": 1073741824 }
 909                     },
 910                     'state': 'Committed',
 911                     'output_name': 'Output for step test_run_True',
 912                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 913                     'output_path': '/var/spool/cwl',
 914                     'output_ttl': 0,
 915                     'container_image': '99999999999999999999999999999993+99',
 916                     'command': ['ls', '/var/spool/cwl'],
 917                     'cwd': '/var/spool/cwl',
 918                     'scheduling_parameters': {},
 919                     'properties': {},
 920                     'secret_mounts': {},
 921                     'output_storage_classes': ["foo_sc", "bar_sc"]
 922                 }))
 923
 924
 925     # The test passes no builder.resources
 926     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 927     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 928     def test_setting_process_properties(self, keepdocker):
 929         arv_docker_clear_cache()
 930
 931         runner = mock.MagicMock()
 932         runner.ignore_docker_for_reuse = False
 933         runner.intermediate_output_ttl = 0
 934         runner.secret_store = cwltool.secrets.SecretStore()
 935         runner.api._rootDesc = {"revision": "20210628"}
 936
 937         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 938         runner.api.collections().get().execute.return_value = {
 939             "portable_data_hash": "99999999999999999999999999999993+99"}
 940
 941         tool = cmap({
 942             "inputs": [
 943                 {"id": "x", "type": "string"}],
 944             "outputs": [],
 945             "baseCommand": "ls",
 946             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 947             "id": "#",
 948             "class": "org.w3id.cwl.cwl.CommandLineTool",
 949             "hints": [
 950             {
 951                 "class": "http://arvados.org/cwl#ProcessProperties",
 952                 "processProperties": [
 953                     {"propertyName": "foo",
 954                      "propertyValue": "bar"},
 955                     {"propertyName": "baz",
 956                      "propertyValue": "$(inputs.x)"},
 957                     {"propertyName": "quux",
 958                      "propertyValue": {
 959                          "q1": 1,
 960                          "q2": 2
 961                      }
 962                     }
 963                 ],
 964             }
 965         ]
 966         })
 967
 968         loadingContext, runtimeContext = self.helper(runner, True)
 969
 970         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 971         arvtool.formatgraph = None
 972
 973         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 974             j.run(runtimeContext)
 975             runner.api.container_requests().create.assert_called_with(
 976                 body=JsonDiffMatcher({
 977                     'environment': {
 978                         'HOME': '/var/spool/cwl',
 979                         'TMPDIR': '/tmp'
 980                     },
 981                     'name': 'test_run_True',
 982                     'runtime_constraints': {
 983                         'vcpus': 1,
 984                         'ram': 1073741824
 985                     },
 986                     'use_existing': True,
 987                     'priority': 500,
 988                     'mounts': {
 989                         '/tmp': {'kind': 'tmp',
 990                                  "capacity": 1073741824
 991                              },
 992                         '/var/spool/cwl': {'kind': 'tmp',
 993                                            "capacity": 1073741824 }
 994                     },
 995                     'state': 'Committed',
 996                     'output_name': 'Output for step test_run_True',
 997                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 998                     'output_path': '/var/spool/cwl',
 999                     'output_ttl': 0,
1000                     'container_image': '99999999999999999999999999999993+99',
1001                     'command': ['ls', '/var/spool/cwl'],
1002                     'cwd': '/var/spool/cwl',
1003                     'scheduling_parameters': {},
1004                     'properties': {
1005                         "baz": "blorp",
1006                         "foo": "bar",
1007                         "quux": {
1008                             "q1": 1,
1009                             "q2": 2
1010                         }
1011                     },
1012                     'secret_mounts': {},
1013                     'output_storage_classes': ["default"]
1014                 }))
1015
1016
1017 class TestWorkflow(unittest.TestCase):
1018     def setUp(self):
1019         cwltool.process._names = set()
1020         arv_docker_clear_cache()
1021
1022     def helper(self, runner, enable_reuse=True):
1023         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1024
1025         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1026                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1027
1028         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1029         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1030         document_loader.fetch_text = document_loader.fetcher.fetch_text
1031         document_loader.check_exists = document_loader.fetcher.check_exists
1032
1033         loadingContext = arvados_cwl.context.ArvLoadingContext(
1034             {"avsc_names": avsc_names,
1035              "basedir": "",
1036              "make_fs_access": make_fs_access,
1037              "loader": document_loader,
1038              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1039              "construct_tool_object": runner.arv_make_tool})
1040         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1041             {"work_api": "containers",
1042              "basedir": "",
1043              "name": "test_run_wf_"+str(enable_reuse),
1044              "make_fs_access": make_fs_access,
1045              "tmpdir": "/tmp",
1046              "enable_reuse": enable_reuse,
1047              "priority": 500})
1048
1049         return loadingContext, runtimeContext
1050
1051     # The test passes no builder.resources
1052     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1053     @mock.patch("arvados.collection.CollectionReader")
1054     @mock.patch("arvados.collection.Collection")
1055     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1056     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1057         arvados_cwl.add_arv_hints()
1058
1059         api = mock.MagicMock()
1060         api._rootDesc = get_rootDesc()
1061
1062         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1063         self.assertEqual(runner.work_api, 'containers')
1064
1065         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1066         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1067         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1068                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1069
1070         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1071         runner.ignore_docker_for_reuse = False
1072         runner.num_retries = 0
1073         runner.secret_store = cwltool.secrets.SecretStore()
1074
1075         loadingContext, runtimeContext = self.helper(runner)
1076         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1077
1078         mockcollectionreader().exists.return_value = True
1079
1080         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1081         metadata["cwlVersion"] = tool["cwlVersion"]
1082
1083         mockc = mock.MagicMock()
1084         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1085         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1086
1087         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1088         arvtool.formatgraph = None
1089         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1090
1091         next(it).run(runtimeContext)
1092         next(it).run(runtimeContext)
1093
1094         with open("tests/wf/scatter2_subwf.cwl") as f:
1095             subwf = StripYAMLComments(f.read()).rstrip()
1096
1097         runner.api.container_requests().create.assert_called_with(
1098             body=JsonDiffMatcher({
1099                 "command": [
1100                     "cwltool",
1101                     "--no-container",
1102                     "--move-outputs",
1103                     "--preserve-entire-environment",
1104                     "workflow.cwl",
1105                     "cwl.input.yml"
1106                 ],
1107                 "container_image": "99999999999999999999999999999993+99",
1108                 "cwd": "/var/spool/cwl",
1109                 "environment": {
1110                     "HOME": "/var/spool/cwl",
1111                     "TMPDIR": "/tmp"
1112                 },
1113                 "mounts": {
1114                     "/keep/99999999999999999999999999999999+118": {
1115                         "kind": "collection",
1116                         "portable_data_hash": "99999999999999999999999999999999+118"
1117                     },
1118                     "/tmp": {
1119                         "capacity": 1073741824,
1120                         "kind": "tmp"
1121                     },
1122                     "/var/spool/cwl": {
1123                         "capacity": 1073741824,
1124                         "kind": "tmp"
1125                     },
1126                     "/var/spool/cwl/cwl.input.yml": {
1127                         "kind": "collection",
1128                         "path": "cwl.input.yml",
1129                         "portable_data_hash": "99999999999999999999999999999996+99"
1130                     },
1131                     "/var/spool/cwl/workflow.cwl": {
1132                         "kind": "collection",
1133                         "path": "workflow.cwl",
1134                         "portable_data_hash": "99999999999999999999999999999996+99"
1135                     },
1136                     "stdout": {
1137                         "kind": "file",
1138                         "path": "/var/spool/cwl/cwl.output.json"
1139                     }
1140                 },
1141                 "name": "scatterstep",
1142                 "output_name": "Output for step scatterstep",
1143                 "output_path": "/var/spool/cwl",
1144                 "output_ttl": 0,
1145                 "priority": 500,
1146                 "properties": {},
1147                 "runtime_constraints": {
1148                     "ram": 1073741824,
1149                     "vcpus": 1
1150                 },
1151                 "scheduling_parameters": {},
1152                 "secret_mounts": {},
1153                 "state": "Committed",
1154                 "use_existing": True,
1155                 'output_storage_classes': ["default"]
1156             }))
1157         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1158         mockc.open().__enter__().write.assert_has_calls([mock.call(
1159 '''{
1160   "fileblub": {
1161     "basename": "token.txt",
1162     "class": "File",
1163     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1164     "size": 0
1165   },
1166   "sleeptime": 5
1167 }''')])
1168
1169     # The test passes no builder.resources
1170     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1171     @mock.patch("arvados.collection.CollectionReader")
1172     @mock.patch("arvados.collection.Collection")
1173     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1174     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1175         arvados_cwl.add_arv_hints()
1176
1177         api = mock.MagicMock()
1178         api._rootDesc = get_rootDesc()
1179
1180         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1181         self.assertEqual(runner.work_api, 'containers')
1182
1183         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1184         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1185                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1186         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1187                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1188
1189         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1190         runner.ignore_docker_for_reuse = False
1191         runner.num_retries = 0
1192         runner.secret_store = cwltool.secrets.SecretStore()
1193
1194         loadingContext, runtimeContext = self.helper(runner)
1195         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1196         loadingContext.do_update = True
1197         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1198
1199         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1200
1201         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1202         arvtool.formatgraph = None
1203         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1204
1205         next(it).run(runtimeContext)
1206         next(it).run(runtimeContext)
1207
1208         with open("tests/wf/echo-subwf.cwl") as f:
1209             subwf = StripYAMLComments(f.read())
1210
1211         runner.api.container_requests().create.assert_called_with(
1212             body=JsonDiffMatcher({
1213                 'output_ttl': 0,
1214                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1215                 'scheduling_parameters': {},
1216                 'name': u'echo-subwf',
1217                 'secret_mounts': {},
1218                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1219                 'properties': {},
1220                 'priority': 500,
1221                 'mounts': {
1222                     '/var/spool/cwl/cwl.input.yml': {
1223                         'portable_data_hash': '99999999999999999999999999999996+99',
1224                         'kind': 'collection',
1225                         'path': 'cwl.input.yml'
1226                     },
1227                     '/var/spool/cwl/workflow.cwl': {
1228                         'portable_data_hash': '99999999999999999999999999999996+99',
1229                         'kind': 'collection',
1230                         'path': 'workflow.cwl'
1231                     },
1232                     'stdout': {
1233                         'path': '/var/spool/cwl/cwl.output.json',
1234                         'kind': 'file'
1235                     },
1236                     '/tmp': {
1237                         'kind': 'tmp',
1238                         'capacity': 1073741824
1239                     }, '/var/spool/cwl': {
1240                         'kind': 'tmp',
1241                         'capacity': 3221225472
1242                     }
1243                 },
1244                 'state': 'Committed',
1245                 'output_path': '/var/spool/cwl',
1246                 'container_image': '99999999999999999999999999999993+99',
1247                 'command': [
1248                     u'cwltool',
1249                     u'--no-container',
1250                     u'--move-outputs',
1251                     u'--preserve-entire-environment',
1252                     u'workflow.cwl',
1253                     u'cwl.input.yml'
1254                 ],
1255                 'use_existing': True,
1256                 'output_name': u'Output for step echo-subwf',
1257                 'cwd': '/var/spool/cwl',
1258                 'output_storage_classes': ["default"]
1259             }))
1260
1261     def test_default_work_api(self):
1262         arvados_cwl.add_arv_hints()
1263
1264         api = mock.MagicMock()
1265         api._rootDesc = copy.deepcopy(get_rootDesc())
1266         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1267         self.assertEqual(runner.work_api, 'containers')