sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 import cwltool.load_tool
  22 from cwltool.update import INTERNAL_VERSION
  23 from schema_salad.ref_resolver import Loader
  24 from schema_salad.sourceline import cmap
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         arv_docker_clear_cache()
  64
  65     def helper(self, runner, enable_reuse=True):
  66         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  67
  68         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  69                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  70         fs_access = mock.MagicMock()
  71         fs_access.exists.return_value = True
  72
  73         loadingContext = arvados_cwl.context.ArvLoadingContext(
  74             {"avsc_names": avsc_names,
  75              "basedir": "",
  76              "make_fs_access": make_fs_access,
  77              "construct_tool_object": runner.arv_make_tool,
  78              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
  79              })
  80         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  81             {"work_api": "containers",
  82              "basedir": "",
  83              "name": "test_run_"+str(enable_reuse),
  84              "make_fs_access": make_fs_access,
  85              "tmpdir": "/tmp",
  86              "enable_reuse": enable_reuse,
  87              "priority": 500,
  88              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  89             })
  90
  91         def make_tool(toolpath_object, loadingContext):
  92             return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
  93         runner.arv_make_tool.side_effect = make_tool
  94
  95         return loadingContext, runtimeContext
  96
  97     # Helper function to set up the ArvCwlExecutor to use the containers api
  98     # and test that the RuntimeStatusLoggingHandler is set up correctly
  99     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 100         api = mock.MagicMock()
 101         api._rootDesc = copy.deepcopy(get_rootDesc())
 102
 103         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 104         # adds the logging handler that will call runtime_status_update() mock
 105         self.assertFalse(gcc_mock.called)
 106         runner = arvados_cwl.ArvCwlExecutor(api)
 107         self.assertEqual(runner.work_api, 'containers')
 108         root_logger = logging.getLogger('')
 109         handlerClasses = [h.__class__ for h in root_logger.handlers]
 110         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 111         return runner
 112
 113     # The test passes no builder.resources
 114     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 115     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 116     def test_run(self, keepdocker):
 117         for enable_reuse in (True, False):
 118             arv_docker_clear_cache()
 119
 120             runner = mock.MagicMock()
 121             runner.ignore_docker_for_reuse = False
 122             runner.intermediate_output_ttl = 0
 123             runner.secret_store = cwltool.secrets.SecretStore()
 124             runner.api._rootDesc = {"revision": "20210628"}
 125
 126             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 127             runner.api.collections().get().execute.return_value = {
 128                 "portable_data_hash": "99999999999999999999999999999993+99"}
 129
 130             tool = cmap({
 131                 "inputs": [],
 132                 "outputs": [],
 133                 "baseCommand": "ls",
 134                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 135                 "id": "#",
 136                 "class": "org.w3id.cwl.cwl.CommandLineTool"
 137             })
 138
 139             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 140
 141             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 142             arvtool.formatgraph = None
 143
 144             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 145                 j.run(runtimeContext)
 146                 runner.api.container_requests().create.assert_called_with(
 147                     body=JsonDiffMatcher({
 148                         'environment': {
 149                             'HOME': '/var/spool/cwl',
 150                             'TMPDIR': '/tmp'
 151                         },
 152                         'name': 'test_run_'+str(enable_reuse),
 153                         'runtime_constraints': {
 154                             'vcpus': 1,
 155                             'ram': 1073741824
 156                         },
 157                         'use_existing': enable_reuse,
 158                         'priority': 500,
 159                         'mounts': {
 160                             '/tmp': {'kind': 'tmp',
 161                                      "capacity": 1073741824
 162                                  },
 163                             '/var/spool/cwl': {'kind': 'tmp',
 164                                                "capacity": 1073741824 }
 165                         },
 166                         'state': 'Committed',
 167                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 168                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 169                         'output_path': '/var/spool/cwl',
 170                         'output_ttl': 0,
 171                         'container_image': '99999999999999999999999999999993+99',
 172                         'command': ['ls', '/var/spool/cwl'],
 173                         'cwd': '/var/spool/cwl',
 174                         'scheduling_parameters': {},
 175                         'properties': {},
 176                         'secret_mounts': {},
 177                         'output_storage_classes': ["default"]
 178                     }))
 179
 180     # The test passes some fields in builder.resources
 181     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 182     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 183     def test_resource_requirements(self, keepdocker):
 184         runner = mock.MagicMock()
 185         runner.ignore_docker_for_reuse = False
 186         runner.intermediate_output_ttl = 3600
 187         runner.secret_store = cwltool.secrets.SecretStore()
 188         runner.api._rootDesc = {"revision": "20210628"}
 189
 190         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 191         runner.api.collections().get().execute.return_value = {
 192             "portable_data_hash": "99999999999999999999999999999993+99"}
 193
 194         tool = cmap({
 195             "inputs": [],
 196             "outputs": [],
 197             "hints": [{
 198                 "class": "ResourceRequirement",
 199                 "coresMin": 3,
 200                 "ramMin": 3000,
 201                 "tmpdirMin": 4000,
 202                 "outdirMin": 5000
 203             }, {
 204                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 205                 "keep_cache": 512
 206             }, {
 207                 "class": "http://arvados.org/cwl#APIRequirement",
 208             }, {
 209                 "class": "http://arvados.org/cwl#PartitionRequirement",
 210                 "partition": "blurb"
 211             }, {
 212                 "class": "http://arvados.org/cwl#IntermediateOutput",
 213                 "outputTTL": 7200
 214             }, {
 215                 "class": "http://arvados.org/cwl#ReuseRequirement",
 216                 "enableReuse": False
 217             }],
 218             "baseCommand": "ls",
 219             "id": "#",
 220             "class": "org.w3id.cwl.cwl.CommandLineTool"
 221         })
 222
 223         loadingContext, runtimeContext = self.helper(runner)
 224         runtimeContext.name = "test_resource_requirements"
 225
 226         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 227         arvtool.formatgraph = None
 228         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 229             j.run(runtimeContext)
 230
 231         call_args, call_kwargs = runner.api.container_requests().create.call_args
 232
 233         call_body_expected = {
 234             'environment': {
 235                 'HOME': '/var/spool/cwl',
 236                 'TMPDIR': '/tmp'
 237             },
 238             'name': 'test_resource_requirements',
 239             'runtime_constraints': {
 240                 'vcpus': 3,
 241                 'ram': 3145728000,
 242                 'keep_cache_ram': 536870912,
 243                 'API': True
 244             },
 245             'use_existing': False,
 246             'priority': 500,
 247             'mounts': {
 248                 '/tmp': {'kind': 'tmp',
 249                          "capacity": 4194304000 },
 250                 '/var/spool/cwl': {'kind': 'tmp',
 251                                    "capacity": 5242880000 }
 252             },
 253             'state': 'Committed',
 254             'output_name': 'Output for step test_resource_requirements',
 255             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 256             'output_path': '/var/spool/cwl',
 257             'output_ttl': 7200,
 258             'container_image': '99999999999999999999999999999993+99',
 259             'command': ['ls'],
 260             'cwd': '/var/spool/cwl',
 261             'scheduling_parameters': {
 262                 'partitions': ['blurb']
 263             },
 264             'properties': {},
 265             'secret_mounts': {},
 266             'output_storage_classes': ["default"]
 267         }
 268
 269         call_body = call_kwargs.get('body', None)
 270         self.assertNotEqual(None, call_body)
 271         for key in call_body:
 272             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 273
 274
 275     # The test passes some fields in builder.resources
 276     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 277     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 278     @mock.patch("arvados.collection.Collection")
 279     def test_initial_work_dir(self, collection_mock, keepdocker):
 280         runner = mock.MagicMock()
 281         runner.ignore_docker_for_reuse = False
 282         runner.intermediate_output_ttl = 0
 283         runner.secret_store = cwltool.secrets.SecretStore()
 284         runner.api._rootDesc = {"revision": "20210628"}
 285
 286         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 287         runner.api.collections().get().execute.return_value = {
 288             "portable_data_hash": "99999999999999999999999999999993+99"}
 289
 290         sourcemock = mock.MagicMock()
 291         def get_collection_mock(p):
 292             if "/" in p:
 293                 return (sourcemock, p.split("/", 1)[1])
 294             else:
 295                 return (sourcemock, "")
 296         runner.fs_access.get_collection.side_effect = get_collection_mock
 297
 298         vwdmock = mock.MagicMock()
 299         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 300
 301         tool = cmap({
 302             "inputs": [],
 303             "outputs": [],
 304             "hints": [{
 305                 "class": "InitialWorkDirRequirement",
 306                 "listing": [{
 307                     "class": "File",
 308                     "basename": "foo",
 309                     "location": "keep:99999999999999999999999999999995+99/bar"
 310                 },
 311                 {
 312                     "class": "Directory",
 313                     "basename": "foo2",
 314                     "location": "keep:99999999999999999999999999999995+99"
 315                 },
 316                 {
 317                     "class": "File",
 318                     "basename": "filename",
 319                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 320                 },
 321                 {
 322                     "class": "Directory",
 323                     "basename": "subdir",
 324                     "location": "keep:99999999999999999999999999999995+99/subdir"
 325                 }                        ]
 326             }],
 327             "baseCommand": "ls",
 328             "class": "CommandLineTool",
 329             "cwlVersion": "v1.2",
 330             "id": ""
 331         })
 332
 333         loadingContext, runtimeContext = self.helper(runner)
 334         runtimeContext.name = "test_initial_work_dir"
 335
 336         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 337
 338         arvtool.formatgraph = None
 339         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 340             j.run(runtimeContext)
 341
 342         call_args, call_kwargs = runner.api.container_requests().create.call_args
 343
 344         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 345         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 346         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 347         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 348
 349         call_body_expected = {
 350             'environment': {
 351                 'HOME': '/var/spool/cwl',
 352                 'TMPDIR': '/tmp'
 353             },
 354             'name': 'test_initial_work_dir',
 355             'runtime_constraints': {
 356                 'vcpus': 1,
 357                 'ram': 268435456
 358             },
 359             'use_existing': True,
 360             'priority': 500,
 361             'mounts': {
 362                 '/tmp': {'kind': 'tmp',
 363                          "capacity": 1073741824 },
 364                 '/var/spool/cwl': {'kind': 'tmp',
 365                                    "capacity": 1073741824 },
 366                 '/var/spool/cwl/foo': {
 367                     'kind': 'collection',
 368                     'path': 'foo',
 369                     'portable_data_hash': '99999999999999999999999999999996+99'
 370                 },
 371                 '/var/spool/cwl/foo2': {
 372                     'kind': 'collection',
 373                     'path': 'foo2',
 374                     'portable_data_hash': '99999999999999999999999999999996+99'
 375                 },
 376                 '/var/spool/cwl/filename': {
 377                     'kind': 'collection',
 378                     'path': 'filename',
 379                     'portable_data_hash': '99999999999999999999999999999996+99'
 380                 },
 381                 '/var/spool/cwl/subdir': {
 382                     'kind': 'collection',
 383                     'path': 'subdir',
 384                     'portable_data_hash': '99999999999999999999999999999996+99'
 385                 }
 386             },
 387             'state': 'Committed',
 388             'output_name': 'Output for step test_initial_work_dir',
 389             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 390             'output_path': '/var/spool/cwl',
 391             'output_ttl': 0,
 392             'container_image': '99999999999999999999999999999993+99',
 393             'command': ['ls'],
 394             'cwd': '/var/spool/cwl',
 395             'scheduling_parameters': {
 396             },
 397             'properties': {},
 398             'secret_mounts': {},
 399             'output_storage_classes': ["default"]
 400         }
 401
 402         call_body = call_kwargs.get('body', None)
 403         self.assertNotEqual(None, call_body)
 404         for key in call_body:
 405             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 406
 407
 408     # Test redirecting stdin/stdout/stderr
 409     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 410     def test_redirects(self, keepdocker):
 411         runner = mock.MagicMock()
 412         runner.ignore_docker_for_reuse = False
 413         runner.intermediate_output_ttl = 0
 414         runner.secret_store = cwltool.secrets.SecretStore()
 415         runner.api._rootDesc = {"revision": "20210628"}
 416
 417         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 418         runner.api.collections().get().execute.return_value = {
 419             "portable_data_hash": "99999999999999999999999999999993+99"}
 420
 421         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 422
 423         tool = cmap({
 424             "inputs": [],
 425             "outputs": [],
 426             "baseCommand": "ls",
 427             "stdout": "stdout.txt",
 428             "stderr": "stderr.txt",
 429             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 430             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 431             "id": "#",
 432             "class": "org.w3id.cwl.cwl.CommandLineTool"
 433         })
 434
 435         loadingContext, runtimeContext = self.helper(runner)
 436         runtimeContext.name = "test_run_redirect"
 437
 438         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 439         arvtool.formatgraph = None
 440         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 441             j.run(runtimeContext)
 442             runner.api.container_requests().create.assert_called_with(
 443                 body=JsonDiffMatcher({
 444                     'environment': {
 445                         'HOME': '/var/spool/cwl',
 446                         'TMPDIR': '/tmp'
 447                     },
 448                     'name': 'test_run_redirect',
 449                     'runtime_constraints': {
 450                         'vcpus': 1,
 451                         'ram': 1073741824
 452                     },
 453                     'use_existing': True,
 454                     'priority': 500,
 455                     'mounts': {
 456                         '/tmp': {'kind': 'tmp',
 457                                  "capacity": 1073741824 },
 458                         '/var/spool/cwl': {'kind': 'tmp',
 459                                            "capacity": 1073741824 },
 460                         "stderr": {
 461                             "kind": "file",
 462                             "path": "/var/spool/cwl/stderr.txt"
 463                         },
 464                         "stdin": {
 465                             "kind": "collection",
 466                             "path": "file.txt",
 467                             "portable_data_hash": "99999999999999999999999999999996+99"
 468                         },
 469                         "stdout": {
 470                             "kind": "file",
 471                             "path": "/var/spool/cwl/stdout.txt"
 472                         },
 473                     },
 474                     'state': 'Committed',
 475                     "output_name": "Output for step test_run_redirect",
 476                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 477                     'output_path': '/var/spool/cwl',
 478                     'output_ttl': 0,
 479                     'container_image': '99999999999999999999999999999993+99',
 480                     'command': ['ls', '/var/spool/cwl'],
 481                     'cwd': '/var/spool/cwl',
 482                     'scheduling_parameters': {},
 483                     'properties': {},
 484                     'secret_mounts': {},
 485                     'output_storage_classes': ["default"]
 486                 }))
 487
 488     @mock.patch("arvados.collection.Collection")
 489     def test_done(self, col):
 490         api = mock.MagicMock()
 491
 492         runner = mock.MagicMock()
 493         runner.api = api
 494         runner.num_retries = 0
 495         runner.ignore_docker_for_reuse = False
 496         runner.intermediate_output_ttl = 0
 497         runner.secret_store = cwltool.secrets.SecretStore()
 498
 499         runner.api.containers().get().execute.return_value = {"state":"Complete",
 500                                                               "output": "abc+123",
 501                                                               "exit_code": 0}
 502
 503         col().open.return_value = []
 504
 505         loadingContext, runtimeContext = self.helper(runner)
 506
 507         arvjob = arvados_cwl.ArvadosContainer(runner,
 508                                               runtimeContext,
 509                                               mock.MagicMock(),
 510                                               {},
 511                                               None,
 512                                               [],
 513                                               [],
 514                                               "testjob")
 515         arvjob.output_callback = mock.MagicMock()
 516         arvjob.collect_outputs = mock.MagicMock()
 517         arvjob.successCodes = [0]
 518         arvjob.outdir = "/var/spool/cwl"
 519         arvjob.output_ttl = 3600
 520
 521         arvjob.collect_outputs.return_value = {"out": "stuff"}
 522
 523         arvjob.done({
 524             "state": "Final",
 525             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 526             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 527             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 528             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 529             "modified_at": "2017-05-26T12:01:22Z"
 530         })
 531
 532         self.assertFalse(api.collections().create.called)
 533         self.assertFalse(runner.runtime_status_error.called)
 534
 535         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 536         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 537         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 538
 539     # Test to make sure we dont call runtime_status_update if we already did
 540     # some where higher up in the call stack
 541     @mock.patch("arvados_cwl.util.get_current_container")
 542     def test_recursive_runtime_status_update(self, gcc_mock):
 543         self.setup_and_test_container_executor_and_logging(gcc_mock)
 544         root_logger = logging.getLogger('')
 545
 546         # get_current_container is invoked when we call runtime_status_update
 547         # so try and log again!
 548         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 549         try:
 550             root_logger.error("First Error")
 551         except RuntimeError:
 552             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 553
 554
 555     # Test to make sure that an exception raised from
 556     # get_current_container doesn't cause the logger to raise an
 557     # exception
 558     @mock.patch("arvados_cwl.util.get_current_container")
 559     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 560         self.setup_and_test_container_executor_and_logging(gcc_mock)
 561         root_logger = logging.getLogger('')
 562
 563         # get_current_container is invoked when we call
 564         # runtime_status_update, it is going to also raise an
 565         # exception.
 566         gcc_mock.side_effect = Exception("Second Error")
 567         try:
 568             root_logger.error("First Error")
 569         except Exception:
 570             self.fail("Exception in logger should not propagate")
 571         self.assertTrue(gcc_mock.called)
 572
 573     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 574     @mock.patch("arvados_cwl.util.get_current_container")
 575     @mock.patch("arvados.collection.CollectionReader")
 576     @mock.patch("arvados.collection.Collection")
 577     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 578         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 579
 580         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 581         self.assertTrue(gcc_mock.called)
 582
 583         runner.num_retries = 0
 584         runner.ignore_docker_for_reuse = False
 585         runner.intermediate_output_ttl = 0
 586         runner.secret_store = cwltool.secrets.SecretStore()
 587         runner.label = mock.MagicMock()
 588         runner.label.return_value = '[container testjob]'
 589
 590         runner.api.containers().get().execute.return_value = {
 591             "state":"Complete",
 592             "output": "abc+123",
 593             "exit_code": 1,
 594             "log": "def+234"
 595         }
 596
 597         col().open.return_value = []
 598
 599         loadingContext, runtimeContext = self.helper(runner)
 600
 601         arvjob = arvados_cwl.ArvadosContainer(runner,
 602                                               runtimeContext,
 603                                               mock.MagicMock(),
 604                                               {},
 605                                               None,
 606                                               [],
 607                                               [],
 608                                               "testjob")
 609         arvjob.output_callback = mock.MagicMock()
 610         arvjob.collect_outputs = mock.MagicMock()
 611         arvjob.successCodes = [0]
 612         arvjob.outdir = "/var/spool/cwl"
 613         arvjob.output_ttl = 3600
 614         arvjob.collect_outputs.return_value = {"out": "stuff"}
 615
 616         arvjob.done({
 617             "state": "Final",
 618             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 619             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 620             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 621             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 622             "modified_at": "2017-05-26T12:01:22Z"
 623         })
 624
 625         rts_mock.assert_called_with(
 626             'error',
 627             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 628             '  ** log is empty **'
 629         )
 630         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 631
 632     # The test passes no builder.resources
 633     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 634     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 635     def test_mounts(self, keepdocker):
 636         runner = mock.MagicMock()
 637         runner.ignore_docker_for_reuse = False
 638         runner.intermediate_output_ttl = 0
 639         runner.secret_store = cwltool.secrets.SecretStore()
 640         runner.api._rootDesc = {"revision": "20210628"}
 641
 642         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 643         runner.api.collections().get().execute.return_value = {
 644             "portable_data_hash": "99999999999999999999999999999994+99",
 645             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 646
 647         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 648
 649         tool = cmap({
 650             "inputs": [
 651                 {"id": "p1",
 652                  "type": "Directory"}
 653             ],
 654             "outputs": [],
 655             "baseCommand": "ls",
 656             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 657             "id": "#",
 658             "class": "org.w3id.cwl.cwl.CommandLineTool"
 659         })
 660
 661         loadingContext, runtimeContext = self.helper(runner)
 662         runtimeContext.name = "test_run_mounts"
 663
 664         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 665         arvtool.formatgraph = None
 666         job_order = {
 667             "p1": {
 668                 "class": "Directory",
 669                 "location": "keep:99999999999999999999999999999994+44",
 670                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 671                 "listing": [
 672                     {
 673                         "class": "File",
 674                         "location": "keep:99999999999999999999999999999994+44/file1",
 675                     },
 676                     {
 677                         "class": "File",
 678                         "location": "keep:99999999999999999999999999999994+44/file2",
 679                     }
 680                 ]
 681             }
 682         }
 683         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 684             j.run(runtimeContext)
 685             runner.api.container_requests().create.assert_called_with(
 686                 body=JsonDiffMatcher({
 687                     'environment': {
 688                         'HOME': '/var/spool/cwl',
 689                         'TMPDIR': '/tmp'
 690                     },
 691                     'name': 'test_run_mounts',
 692                     'runtime_constraints': {
 693                         'vcpus': 1,
 694                         'ram': 1073741824
 695                     },
 696                     'use_existing': True,
 697                     'priority': 500,
 698                     'mounts': {
 699                         "/keep/99999999999999999999999999999994+44": {
 700                             "kind": "collection",
 701                             "portable_data_hash": "99999999999999999999999999999994+44",
 702                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 703                         },
 704                         '/tmp': {'kind': 'tmp',
 705                                  "capacity": 1073741824 },
 706                         '/var/spool/cwl': {'kind': 'tmp',
 707                                            "capacity": 1073741824 }
 708                     },
 709                     'state': 'Committed',
 710                     'output_name': 'Output for step test_run_mounts',
 711                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 712                     'output_path': '/var/spool/cwl',
 713                     'output_ttl': 0,
 714                     'container_image': '99999999999999999999999999999994+99',
 715                     'command': ['ls', '/var/spool/cwl'],
 716                     'cwd': '/var/spool/cwl',
 717                     'scheduling_parameters': {},
 718                     'properties': {},
 719                     'secret_mounts': {},
 720                     'output_storage_classes': ["default"]
 721                 }))
 722
 723     # The test passes no builder.resources
 724     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 725     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 726     def test_secrets(self, keepdocker):
 727         runner = mock.MagicMock()
 728         runner.ignore_docker_for_reuse = False
 729         runner.intermediate_output_ttl = 0
 730         runner.secret_store = cwltool.secrets.SecretStore()
 731         runner.api._rootDesc = {"revision": "20210628"}
 732
 733         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 734         runner.api.collections().get().execute.return_value = {
 735             "portable_data_hash": "99999999999999999999999999999993+99"}
 736
 737         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 738
 739         tool = cmap({"arguments": ["md5sum", "example.conf"],
 740                      "class": "org.w3id.cwl.cwl.CommandLineTool",
 741                      "hints": [
 742                          {
 743                              "class": "http://commonwl.org/cwltool#Secrets",
 744                              "secrets": [
 745                                  "#secret_job.cwl/pw"
 746                              ]
 747                          }
 748                      ],
 749                      "id": "#secret_job.cwl",
 750                      "inputs": [
 751                          {
 752                              "id": "#secret_job.cwl/pw",
 753                              "type": "string"
 754                          }
 755                      ],
 756                      "outputs": [
 757                      ],
 758                      "requirements": [
 759                          {
 760                              "class": "InitialWorkDirRequirement",
 761                              "listing": [
 762                                  {
 763                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 764                                      "entryname": "example.conf"
 765                                  }
 766                              ]
 767                          }
 768                      ]})
 769
 770         loadingContext, runtimeContext = self.helper(runner)
 771         runtimeContext.name = "test_secrets"
 772
 773         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 774         arvtool.formatgraph = None
 775
 776         job_order = {"pw": "blorp"}
 777         runner.secret_store.store(["pw"], job_order)
 778
 779         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 780             j.run(runtimeContext)
 781             runner.api.container_requests().create.assert_called_with(
 782                 body=JsonDiffMatcher({
 783                     'environment': {
 784                         'HOME': '/var/spool/cwl',
 785                         'TMPDIR': '/tmp'
 786                     },
 787                     'name': 'test_secrets',
 788                     'runtime_constraints': {
 789                         'vcpus': 1,
 790                         'ram': 1073741824
 791                     },
 792                     'use_existing': True,
 793                     'priority': 500,
 794                     'mounts': {
 795                         '/tmp': {'kind': 'tmp',
 796                                  "capacity": 1073741824
 797                              },
 798                         '/var/spool/cwl': {'kind': 'tmp',
 799                                            "capacity": 1073741824 }
 800                     },
 801                     'state': 'Committed',
 802                     'output_name': 'Output for step test_secrets',
 803                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 804                     'output_path': '/var/spool/cwl',
 805                     'output_ttl': 0,
 806                     'container_image': '99999999999999999999999999999993+99',
 807                     'command': ['md5sum', 'example.conf'],
 808                     'cwd': '/var/spool/cwl',
 809                     'scheduling_parameters': {},
 810                     'properties': {},
 811                     "secret_mounts": {
 812                         "/var/spool/cwl/example.conf": {
 813                             "content": "username: user\npassword: blorp\n",
 814                             "kind": "text"
 815                         }
 816                     },
 817                     'output_storage_classes': ["default"]
 818                 }))
 819
 820     # The test passes no builder.resources
 821     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 822     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 823     def test_timelimit(self, keepdocker):
 824         runner = mock.MagicMock()
 825         runner.ignore_docker_for_reuse = False
 826         runner.intermediate_output_ttl = 0
 827         runner.secret_store = cwltool.secrets.SecretStore()
 828         runner.api._rootDesc = {"revision": "20210628"}
 829
 830         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 831         runner.api.collections().get().execute.return_value = {
 832             "portable_data_hash": "99999999999999999999999999999993+99"}
 833
 834         tool = cmap({
 835             "inputs": [],
 836             "outputs": [],
 837             "baseCommand": "ls",
 838             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 839             "id": "#",
 840             "class": "org.w3id.cwl.cwl.CommandLineTool",
 841             "hints": [
 842                 {
 843                     "class": "ToolTimeLimit",
 844                     "timelimit": 42
 845                 }
 846             ]
 847         })
 848
 849         loadingContext, runtimeContext = self.helper(runner)
 850         runtimeContext.name = "test_timelimit"
 851
 852         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 853         arvtool.formatgraph = None
 854
 855         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 856             j.run(runtimeContext)
 857
 858         _, kwargs = runner.api.container_requests().create.call_args
 859         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 860
 861
 862     # The test passes no builder.resources
 863     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 864     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 865     def test_setting_storage_class(self, keepdocker):
 866         arv_docker_clear_cache()
 867
 868         runner = mock.MagicMock()
 869         runner.ignore_docker_for_reuse = False
 870         runner.intermediate_output_ttl = 0
 871         runner.secret_store = cwltool.secrets.SecretStore()
 872         runner.api._rootDesc = {"revision": "20210628"}
 873
 874         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 875         runner.api.collections().get().execute.return_value = {
 876             "portable_data_hash": "99999999999999999999999999999993+99"}
 877
 878         tool = cmap({
 879             "inputs": [],
 880             "outputs": [],
 881             "baseCommand": "ls",
 882             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 883             "id": "#",
 884             "class": "org.w3id.cwl.cwl.CommandLineTool",
 885             "hints": [
 886                 {
 887                     "class": "http://arvados.org/cwl#OutputStorageClass",
 888                     "finalStorageClass": ["baz_sc", "qux_sc"],
 889                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 890                 }
 891             ]
 892         })
 893
 894         loadingContext, runtimeContext = self.helper(runner, True)
 895
 896         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 897         arvtool.formatgraph = None
 898
 899         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 900             j.run(runtimeContext)
 901             runner.api.container_requests().create.assert_called_with(
 902                 body=JsonDiffMatcher({
 903                     'environment': {
 904                         'HOME': '/var/spool/cwl',
 905                         'TMPDIR': '/tmp'
 906                     },
 907                     'name': 'test_run_True',
 908                     'runtime_constraints': {
 909                         'vcpus': 1,
 910                         'ram': 1073741824
 911                     },
 912                     'use_existing': True,
 913                     'priority': 500,
 914                     'mounts': {
 915                         '/tmp': {'kind': 'tmp',
 916                                  "capacity": 1073741824
 917                              },
 918                         '/var/spool/cwl': {'kind': 'tmp',
 919                                            "capacity": 1073741824 }
 920                     },
 921                     'state': 'Committed',
 922                     'output_name': 'Output for step test_run_True',
 923                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 924                     'output_path': '/var/spool/cwl',
 925                     'output_ttl': 0,
 926                     'container_image': '99999999999999999999999999999993+99',
 927                     'command': ['ls', '/var/spool/cwl'],
 928                     'cwd': '/var/spool/cwl',
 929                     'scheduling_parameters': {},
 930                     'properties': {},
 931                     'secret_mounts': {},
 932                     'output_storage_classes': ["foo_sc", "bar_sc"]
 933                 }))
 934
 935
 936     # The test passes no builder.resources
 937     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 938     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 939     def test_setting_process_properties(self, keepdocker):
 940         arv_docker_clear_cache()
 941
 942         runner = mock.MagicMock()
 943         runner.ignore_docker_for_reuse = False
 944         runner.intermediate_output_ttl = 0
 945         runner.secret_store = cwltool.secrets.SecretStore()
 946         runner.api._rootDesc = {"revision": "20210628"}
 947
 948         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 949         runner.api.collections().get().execute.return_value = {
 950             "portable_data_hash": "99999999999999999999999999999993+99"}
 951
 952         tool = cmap({
 953             "inputs": [
 954                 {"id": "x", "type": "string"}],
 955             "outputs": [],
 956             "baseCommand": "ls",
 957             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 958             "id": "#",
 959             "class": "org.w3id.cwl.cwl.CommandLineTool",
 960             "hints": [
 961             {
 962                 "class": "http://arvados.org/cwl#ProcessProperties",
 963                 "processProperties": [
 964                     {"propertyName": "foo",
 965                      "propertyValue": "bar"},
 966                     {"propertyName": "baz",
 967                      "propertyValue": "$(inputs.x)"},
 968                     {"propertyName": "quux",
 969                      "propertyValue": {
 970                          "q1": 1,
 971                          "q2": 2
 972                      }
 973                     }
 974                 ],
 975             }
 976         ]
 977         })
 978
 979         loadingContext, runtimeContext = self.helper(runner, True)
 980
 981         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 982         arvtool.formatgraph = None
 983
 984         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 985             j.run(runtimeContext)
 986             runner.api.container_requests().create.assert_called_with(
 987                 body=JsonDiffMatcher({
 988                     'environment': {
 989                         'HOME': '/var/spool/cwl',
 990                         'TMPDIR': '/tmp'
 991                     },
 992                     'name': 'test_run_True',
 993                     'runtime_constraints': {
 994                         'vcpus': 1,
 995                         'ram': 1073741824
 996                     },
 997                     'use_existing': True,
 998                     'priority': 500,
 999                     'mounts': {
1000                         '/tmp': {'kind': 'tmp',
1001                                  "capacity": 1073741824
1002                              },
1003                         '/var/spool/cwl': {'kind': 'tmp',
1004                                            "capacity": 1073741824 }
1005                     },
1006                     'state': 'Committed',
1007                     'output_name': 'Output for step test_run_True',
1008                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1009                     'output_path': '/var/spool/cwl',
1010                     'output_ttl': 0,
1011                     'container_image': '99999999999999999999999999999993+99',
1012                     'command': ['ls', '/var/spool/cwl'],
1013                     'cwd': '/var/spool/cwl',
1014                     'scheduling_parameters': {},
1015                     'properties': {
1016                         "baz": "blorp",
1017                         "foo": "bar",
1018                         "quux": {
1019                             "q1": 1,
1020                             "q2": 2
1021                         }
1022                     },
1023                     'secret_mounts': {},
1024                     'output_storage_classes': ["default"]
1025                 }))
1026
1027
1028     # The test passes no builder.resources
1029     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1030     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1031     def test_cuda_requirement(self, keepdocker):
1032         arv_docker_clear_cache()
1033
1034         runner = mock.MagicMock()
1035         runner.ignore_docker_for_reuse = False
1036         runner.intermediate_output_ttl = 0
1037         runner.secret_store = cwltool.secrets.SecretStore()
1038         runner.api._rootDesc = {"revision": "20210628"}
1039
1040         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1041         runner.api.collections().get().execute.return_value = {
1042             "portable_data_hash": "99999999999999999999999999999993+99"}
1043
1044         tool = cmap({
1045             "inputs": [],
1046             "outputs": [],
1047             "baseCommand": "nvidia-smi",
1048             "arguments": [],
1049             "id": "#",
1050             "class": "org.w3id.cwl.cwl.CommandLineTool",
1051             "hints": [
1052             {
1053                 "class": "http://arvados.org/cwl#CUDARequirement",
1054                 "minCUDADriverVersion": "11.0",
1055                 "minCUDAHardwareCapability": "9.0",
1056             }
1057         ]
1058         })
1059
1060         loadingContext, runtimeContext = self.helper(runner, True)
1061
1062         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1063         arvtool.formatgraph = None
1064
1065         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1066             j.run(runtimeContext)
1067             runner.api.container_requests().create.assert_called_with(
1068                 body=JsonDiffMatcher({
1069                     'environment': {
1070                         'HOME': '/var/spool/cwl',
1071                         'TMPDIR': '/tmp'
1072                     },
1073                     'name': 'test_run_True',
1074                     'runtime_constraints': {
1075                         'vcpus': 1,
1076                         'ram': 1073741824,
1077                         'cuda': {
1078                             'device_count': 1,
1079                             'driver_version': "11.0",
1080                             'hardware_capability': "9.0"
1081                         }
1082                     },
1083                     'use_existing': True,
1084                     'priority': 500,
1085                     'mounts': {
1086                         '/tmp': {'kind': 'tmp',
1087                                  "capacity": 1073741824
1088                              },
1089                         '/var/spool/cwl': {'kind': 'tmp',
1090                                            "capacity": 1073741824 }
1091                     },
1092                     'state': 'Committed',
1093                     'output_name': 'Output for step test_run_True',
1094                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1095                     'output_path': '/var/spool/cwl',
1096                     'output_ttl': 0,
1097                     'container_image': '99999999999999999999999999999993+99',
1098                     'command': ['nvidia-smi'],
1099                     'cwd': '/var/spool/cwl',
1100                     'scheduling_parameters': {},
1101                     'properties': {},
1102                     'secret_mounts': {},
1103                     'output_storage_classes': ["default"]
1104                 }))
1105
1106
1107 class TestWorkflow(unittest.TestCase):
1108     def setUp(self):
1109         cwltool.process._names = set()
1110         arv_docker_clear_cache()
1111
1112     def helper(self, runner, enable_reuse=True):
1113         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1114
1115         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1116                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1117
1118         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1119         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1120         document_loader.fetch_text = document_loader.fetcher.fetch_text
1121         document_loader.check_exists = document_loader.fetcher.check_exists
1122
1123         loadingContext = arvados_cwl.context.ArvLoadingContext(
1124             {"avsc_names": avsc_names,
1125              "basedir": "",
1126              "make_fs_access": make_fs_access,
1127              "loader": document_loader,
1128              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1129              "construct_tool_object": runner.arv_make_tool})
1130         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1131             {"work_api": "containers",
1132              "basedir": "",
1133              "name": "test_run_wf_"+str(enable_reuse),
1134              "make_fs_access": make_fs_access,
1135              "tmpdir": "/tmp",
1136              "enable_reuse": enable_reuse,
1137              "priority": 500})
1138
1139         return loadingContext, runtimeContext
1140
1141     # The test passes no builder.resources
1142     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1143     @mock.patch("arvados.collection.CollectionReader")
1144     @mock.patch("arvados.collection.Collection")
1145     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1146     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1147         arvados_cwl.add_arv_hints()
1148
1149         api = mock.MagicMock()
1150         api._rootDesc = get_rootDesc()
1151
1152         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1153         self.assertEqual(runner.work_api, 'containers')
1154
1155         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1156         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1157         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1158                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1159
1160         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1161         runner.ignore_docker_for_reuse = False
1162         runner.num_retries = 0
1163         runner.secret_store = cwltool.secrets.SecretStore()
1164
1165         loadingContext, runtimeContext = self.helper(runner)
1166         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1167
1168         mockcollectionreader().exists.return_value = True
1169
1170         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1171         metadata["cwlVersion"] = tool["cwlVersion"]
1172
1173         mockc = mock.MagicMock()
1174         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1175         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1176
1177         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1178         arvtool.formatgraph = None
1179         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1180
1181         next(it).run(runtimeContext)
1182         next(it).run(runtimeContext)
1183
1184         with open("tests/wf/scatter2_subwf.cwl") as f:
1185             subwf = StripYAMLComments(f.read()).rstrip()
1186
1187         runner.api.container_requests().create.assert_called_with(
1188             body=JsonDiffMatcher({
1189                 "command": [
1190                     "cwltool",
1191                     "--no-container",
1192                     "--move-outputs",
1193                     "--preserve-entire-environment",
1194                     "workflow.cwl",
1195                     "cwl.input.yml"
1196                 ],
1197                 "container_image": "99999999999999999999999999999993+99",
1198                 "cwd": "/var/spool/cwl",
1199                 "environment": {
1200                     "HOME": "/var/spool/cwl",
1201                     "TMPDIR": "/tmp"
1202                 },
1203                 "mounts": {
1204                     "/keep/99999999999999999999999999999999+118": {
1205                         "kind": "collection",
1206                         "portable_data_hash": "99999999999999999999999999999999+118"
1207                     },
1208                     "/tmp": {
1209                         "capacity": 1073741824,
1210                         "kind": "tmp"
1211                     },
1212                     "/var/spool/cwl": {
1213                         "capacity": 1073741824,
1214                         "kind": "tmp"
1215                     },
1216                     "/var/spool/cwl/cwl.input.yml": {
1217                         "kind": "collection",
1218                         "path": "cwl.input.yml",
1219                         "portable_data_hash": "99999999999999999999999999999996+99"
1220                     },
1221                     "/var/spool/cwl/workflow.cwl": {
1222                         "kind": "collection",
1223                         "path": "workflow.cwl",
1224                         "portable_data_hash": "99999999999999999999999999999996+99"
1225                     },
1226                     "stdout": {
1227                         "kind": "file",
1228                         "path": "/var/spool/cwl/cwl.output.json"
1229                     }
1230                 },
1231                 "name": "scatterstep",
1232                 "output_name": "Output for step scatterstep",
1233                 "output_path": "/var/spool/cwl",
1234                 "output_ttl": 0,
1235                 "priority": 500,
1236                 "properties": {},
1237                 "runtime_constraints": {
1238                     "ram": 1073741824,
1239                     "vcpus": 1
1240                 },
1241                 "scheduling_parameters": {},
1242                 "secret_mounts": {},
1243                 "state": "Committed",
1244                 "use_existing": True,
1245                 'output_storage_classes': ["default"]
1246             }))
1247         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1248         mockc.open().__enter__().write.assert_has_calls([mock.call(
1249 '''{
1250   "fileblub": {
1251     "basename": "token.txt",
1252     "class": "File",
1253     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1254     "size": 0
1255   },
1256   "sleeptime": 5
1257 }''')])
1258
1259     # The test passes no builder.resources
1260     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1261     @mock.patch("arvados.collection.CollectionReader")
1262     @mock.patch("arvados.collection.Collection")
1263     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1264     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1265         arvados_cwl.add_arv_hints()
1266
1267         api = mock.MagicMock()
1268         api._rootDesc = get_rootDesc()
1269
1270         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1271         self.assertEqual(runner.work_api, 'containers')
1272
1273         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1274         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1275                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1276         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1277                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1278
1279         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1280         runner.ignore_docker_for_reuse = False
1281         runner.num_retries = 0
1282         runner.secret_store = cwltool.secrets.SecretStore()
1283
1284         loadingContext, runtimeContext = self.helper(runner)
1285         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1286         loadingContext.do_update = True
1287         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1288
1289         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1290
1291         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1292         arvtool.formatgraph = None
1293         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1294
1295         next(it).run(runtimeContext)
1296         next(it).run(runtimeContext)
1297
1298         with open("tests/wf/echo-subwf.cwl") as f:
1299             subwf = StripYAMLComments(f.read())
1300
1301         runner.api.container_requests().create.assert_called_with(
1302             body=JsonDiffMatcher({
1303                 'output_ttl': 0,
1304                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1305                 'scheduling_parameters': {},
1306                 'name': u'echo-subwf',
1307                 'secret_mounts': {},
1308                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1309                 'properties': {},
1310                 'priority': 500,
1311                 'mounts': {
1312                     '/var/spool/cwl/cwl.input.yml': {
1313                         'portable_data_hash': '99999999999999999999999999999996+99',
1314                         'kind': 'collection',
1315                         'path': 'cwl.input.yml'
1316                     },
1317                     '/var/spool/cwl/workflow.cwl': {
1318                         'portable_data_hash': '99999999999999999999999999999996+99',
1319                         'kind': 'collection',
1320                         'path': 'workflow.cwl'
1321                     },
1322                     'stdout': {
1323                         'path': '/var/spool/cwl/cwl.output.json',
1324                         'kind': 'file'
1325                     },
1326                     '/tmp': {
1327                         'kind': 'tmp',
1328                         'capacity': 1073741824
1329                     }, '/var/spool/cwl': {
1330                         'kind': 'tmp',
1331                         'capacity': 3221225472
1332                     }
1333                 },
1334                 'state': 'Committed',
1335                 'output_path': '/var/spool/cwl',
1336                 'container_image': '99999999999999999999999999999993+99',
1337                 'command': [
1338                     u'cwltool',
1339                     u'--no-container',
1340                     u'--move-outputs',
1341                     u'--preserve-entire-environment',
1342                     u'workflow.cwl',
1343                     u'cwl.input.yml'
1344                 ],
1345                 'use_existing': True,
1346                 'output_name': u'Output for step echo-subwf',
1347                 'cwd': '/var/spool/cwl',
1348                 'output_storage_classes': ["default"]
1349             }))
1350
1351     def test_default_work_api(self):
1352         arvados_cwl.add_arv_hints()
1353
1354         api = mock.MagicMock()
1355         api._rootDesc = copy.deepcopy(get_rootDesc())
1356         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1357         self.assertEqual(runner.work_api, 'containers')