sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 import cwltool.load_tool
  22 from cwltool.update import INTERNAL_VERSION
  23 from schema_salad.ref_resolver import Loader
  24 from schema_salad.sourceline import cmap
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         arv_docker_clear_cache()
  64
  65     def helper(self, runner, enable_reuse=True):
  66         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  67
  68         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  69                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  70         fs_access = mock.MagicMock()
  71         fs_access.exists.return_value = True
  72
  73         loadingContext = arvados_cwl.context.ArvLoadingContext(
  74             {"avsc_names": avsc_names,
  75              "basedir": "",
  76              "make_fs_access": make_fs_access,
  77              "construct_tool_object": runner.arv_make_tool,
  78              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
  79              })
  80         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  81             {"work_api": "containers",
  82              "basedir": "",
  83              "name": "test_run_"+str(enable_reuse),
  84              "make_fs_access": make_fs_access,
  85              "tmpdir": "/tmp",
  86              "enable_reuse": enable_reuse,
  87              "priority": 500,
  88              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  89             })
  90
  91         if isinstance(runner, mock.MagicMock):
  92             def make_tool(toolpath_object, loadingContext):
  93                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
  94             runner.arv_make_tool.side_effect = make_tool
  95
  96         return loadingContext, runtimeContext
  97
  98     # Helper function to set up the ArvCwlExecutor to use the containers api
  99     # and test that the RuntimeStatusLoggingHandler is set up correctly
 100     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 101         api = mock.MagicMock()
 102         api._rootDesc = copy.deepcopy(get_rootDesc())
 103
 104         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 105         # adds the logging handler that will call runtime_status_update() mock
 106         self.assertFalse(gcc_mock.called)
 107         runner = arvados_cwl.ArvCwlExecutor(api)
 108         self.assertEqual(runner.work_api, 'containers')
 109         root_logger = logging.getLogger('')
 110         handlerClasses = [h.__class__ for h in root_logger.handlers]
 111         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 112         return runner
 113
 114     # The test passes no builder.resources
 115     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 116     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 117     def test_run(self, keepdocker):
 118         for enable_reuse in (True, False):
 119             arv_docker_clear_cache()
 120
 121             runner = mock.MagicMock()
 122             runner.ignore_docker_for_reuse = False
 123             runner.intermediate_output_ttl = 0
 124             runner.secret_store = cwltool.secrets.SecretStore()
 125             runner.api._rootDesc = {"revision": "20210628"}
 126
 127             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 128             runner.api.collections().get().execute.return_value = {
 129                 "portable_data_hash": "99999999999999999999999999999993+99"}
 130
 131             tool = cmap({
 132                 "inputs": [],
 133                 "outputs": [],
 134                 "baseCommand": "ls",
 135                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 136                 "id": "",
 137                 "class": "CommandLineTool",
 138                 "cwlVersion": "v1.2"
 139             })
 140
 141             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 142
 143             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 144             arvtool.formatgraph = None
 145
 146             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 147                 j.run(runtimeContext)
 148                 runner.api.container_requests().create.assert_called_with(
 149                     body=JsonDiffMatcher({
 150                         'environment': {
 151                             'HOME': '/var/spool/cwl',
 152                             'TMPDIR': '/tmp'
 153                         },
 154                         'name': 'test_run_'+str(enable_reuse),
 155                         'runtime_constraints': {
 156                             'vcpus': 1,
 157                             'ram': 268435456
 158                         },
 159                         'use_existing': enable_reuse,
 160                         'priority': 500,
 161                         'mounts': {
 162                             '/tmp': {'kind': 'tmp',
 163                                      "capacity": 1073741824
 164                                  },
 165                             '/var/spool/cwl': {'kind': 'tmp',
 166                                                "capacity": 1073741824 }
 167                         },
 168                         'state': 'Committed',
 169                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 170                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 171                         'output_path': '/var/spool/cwl',
 172                         'output_ttl': 0,
 173                         'container_image': '99999999999999999999999999999993+99',
 174                         'command': ['ls', '/var/spool/cwl'],
 175                         'cwd': '/var/spool/cwl',
 176                         'scheduling_parameters': {},
 177                         'properties': {},
 178                         'secret_mounts': {},
 179                         'output_storage_classes': ["default"]
 180                     }))
 181
 182     # The test passes some fields in builder.resources
 183     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 184     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 185     def test_resource_requirements(self, keepdocker):
 186         arvados_cwl.add_arv_hints()
 187         runner = mock.MagicMock()
 188         runner.ignore_docker_for_reuse = False
 189         runner.intermediate_output_ttl = 3600
 190         runner.secret_store = cwltool.secrets.SecretStore()
 191         runner.api._rootDesc = {"revision": "20210628"}
 192
 193         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 194         runner.api.collections().get().execute.return_value = {
 195             "portable_data_hash": "99999999999999999999999999999993+99"}
 196
 197         tool = cmap({
 198             "inputs": [],
 199             "outputs": [],
 200             "hints": [{
 201                 "class": "ResourceRequirement",
 202                 "coresMin": 3,
 203                 "ramMin": 3000,
 204                 "tmpdirMin": 4000,
 205                 "outdirMin": 5000
 206             }, {
 207                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 208                 "keep_cache": 512
 209             }, {
 210                 "class": "http://arvados.org/cwl#APIRequirement",
 211             }, {
 212                 "class": "http://arvados.org/cwl#PartitionRequirement",
 213                 "partition": "blurb"
 214             }, {
 215                 "class": "http://arvados.org/cwl#IntermediateOutput",
 216                 "outputTTL": 7200
 217             }, {
 218                 "class": "WorkReuse",
 219                 "enableReuse": False
 220             }],
 221             "baseCommand": "ls",
 222             "id": "",
 223             "class": "CommandLineTool",
 224             "cwlVersion": "v1.2"
 225         })
 226
 227         loadingContext, runtimeContext = self.helper(runner)
 228         runtimeContext.name = "test_resource_requirements"
 229
 230         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 231         arvtool.formatgraph = None
 232         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 233             j.run(runtimeContext)
 234
 235         call_args, call_kwargs = runner.api.container_requests().create.call_args
 236
 237         call_body_expected = {
 238             'environment': {
 239                 'HOME': '/var/spool/cwl',
 240                 'TMPDIR': '/tmp'
 241             },
 242             'name': 'test_resource_requirements',
 243             'runtime_constraints': {
 244                 'vcpus': 3,
 245                 'ram': 3145728000,
 246                 'keep_cache_ram': 536870912,
 247                 'API': True
 248             },
 249             'use_existing': False,
 250             'priority': 500,
 251             'mounts': {
 252                 '/tmp': {'kind': 'tmp',
 253                          "capacity": 4194304000 },
 254                 '/var/spool/cwl': {'kind': 'tmp',
 255                                    "capacity": 5242880000 }
 256             },
 257             'state': 'Committed',
 258             'output_name': 'Output for step test_resource_requirements',
 259             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 260             'output_path': '/var/spool/cwl',
 261             'output_ttl': 7200,
 262             'container_image': '99999999999999999999999999999993+99',
 263             'command': ['ls'],
 264             'cwd': '/var/spool/cwl',
 265             'scheduling_parameters': {
 266                 'partitions': ['blurb']
 267             },
 268             'properties': {},
 269             'secret_mounts': {},
 270             'output_storage_classes': ["default"]
 271         }
 272
 273         call_body = call_kwargs.get('body', None)
 274         self.assertNotEqual(None, call_body)
 275         for key in call_body:
 276             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 277
 278
 279     # The test passes some fields in builder.resources
 280     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 281     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 282     @mock.patch("arvados.collection.Collection")
 283     def test_initial_work_dir(self, collection_mock, keepdocker):
 284         runner = mock.MagicMock()
 285         runner.ignore_docker_for_reuse = False
 286         runner.intermediate_output_ttl = 0
 287         runner.secret_store = cwltool.secrets.SecretStore()
 288         runner.api._rootDesc = {"revision": "20210628"}
 289
 290         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 291         runner.api.collections().get().execute.return_value = {
 292             "portable_data_hash": "99999999999999999999999999999993+99"}
 293
 294         sourcemock = mock.MagicMock()
 295         def get_collection_mock(p):
 296             if "/" in p:
 297                 return (sourcemock, p.split("/", 1)[1])
 298             else:
 299                 return (sourcemock, "")
 300         runner.fs_access.get_collection.side_effect = get_collection_mock
 301
 302         vwdmock = mock.MagicMock()
 303         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 304
 305         tool = cmap({
 306             "inputs": [],
 307             "outputs": [],
 308             "hints": [{
 309                 "class": "InitialWorkDirRequirement",
 310                 "listing": [{
 311                     "class": "File",
 312                     "basename": "foo",
 313                     "location": "keep:99999999999999999999999999999995+99/bar"
 314                 },
 315                 {
 316                     "class": "Directory",
 317                     "basename": "foo2",
 318                     "location": "keep:99999999999999999999999999999995+99"
 319                 },
 320                 {
 321                     "class": "File",
 322                     "basename": "filename",
 323                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 324                 },
 325                 {
 326                     "class": "Directory",
 327                     "basename": "subdir",
 328                     "location": "keep:99999999999999999999999999999995+99/subdir"
 329                 }                        ]
 330             }],
 331             "baseCommand": "ls",
 332             "class": "CommandLineTool",
 333             "cwlVersion": "v1.2",
 334             "id": ""
 335         })
 336
 337         loadingContext, runtimeContext = self.helper(runner)
 338         runtimeContext.name = "test_initial_work_dir"
 339
 340         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 341
 342         arvtool.formatgraph = None
 343         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 344             j.run(runtimeContext)
 345
 346         call_args, call_kwargs = runner.api.container_requests().create.call_args
 347
 348         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 349         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 350         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 351         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 352
 353         call_body_expected = {
 354             'environment': {
 355                 'HOME': '/var/spool/cwl',
 356                 'TMPDIR': '/tmp'
 357             },
 358             'name': 'test_initial_work_dir',
 359             'runtime_constraints': {
 360                 'vcpus': 1,
 361                 'ram': 268435456
 362             },
 363             'use_existing': True,
 364             'priority': 500,
 365             'mounts': {
 366                 '/tmp': {'kind': 'tmp',
 367                          "capacity": 1073741824 },
 368                 '/var/spool/cwl': {'kind': 'tmp',
 369                                    "capacity": 1073741824 },
 370                 '/var/spool/cwl/foo': {
 371                     'kind': 'collection',
 372                     'path': 'foo',
 373                     'portable_data_hash': '99999999999999999999999999999996+99'
 374                 },
 375                 '/var/spool/cwl/foo2': {
 376                     'kind': 'collection',
 377                     'path': 'foo2',
 378                     'portable_data_hash': '99999999999999999999999999999996+99'
 379                 },
 380                 '/var/spool/cwl/filename': {
 381                     'kind': 'collection',
 382                     'path': 'filename',
 383                     'portable_data_hash': '99999999999999999999999999999996+99'
 384                 },
 385                 '/var/spool/cwl/subdir': {
 386                     'kind': 'collection',
 387                     'path': 'subdir',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 }
 390             },
 391             'state': 'Committed',
 392             'output_name': 'Output for step test_initial_work_dir',
 393             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 394             'output_path': '/var/spool/cwl',
 395             'output_ttl': 0,
 396             'container_image': '99999999999999999999999999999993+99',
 397             'command': ['ls'],
 398             'cwd': '/var/spool/cwl',
 399             'scheduling_parameters': {
 400             },
 401             'properties': {},
 402             'secret_mounts': {},
 403             'output_storage_classes': ["default"]
 404         }
 405
 406         call_body = call_kwargs.get('body', None)
 407         self.assertNotEqual(None, call_body)
 408         for key in call_body:
 409             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 410
 411
 412     # Test redirecting stdin/stdout/stderr
 413     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 414     def test_redirects(self, keepdocker):
 415         runner = mock.MagicMock()
 416         runner.ignore_docker_for_reuse = False
 417         runner.intermediate_output_ttl = 0
 418         runner.secret_store = cwltool.secrets.SecretStore()
 419         runner.api._rootDesc = {"revision": "20210628"}
 420
 421         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 422         runner.api.collections().get().execute.return_value = {
 423             "portable_data_hash": "99999999999999999999999999999993+99"}
 424
 425         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 426
 427         tool = cmap({
 428             "inputs": [],
 429             "outputs": [],
 430             "baseCommand": "ls",
 431             "stdout": "stdout.txt",
 432             "stderr": "stderr.txt",
 433             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 434             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 435             "id": "",
 436             "class": "CommandLineTool",
 437             "cwlVersion": "v1.2"
 438         })
 439
 440         loadingContext, runtimeContext = self.helper(runner)
 441         runtimeContext.name = "test_run_redirect"
 442
 443         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 444         arvtool.formatgraph = None
 445         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 446             j.run(runtimeContext)
 447             runner.api.container_requests().create.assert_called_with(
 448                 body=JsonDiffMatcher({
 449                     'environment': {
 450                         'HOME': '/var/spool/cwl',
 451                         'TMPDIR': '/tmp'
 452                     },
 453                     'name': 'test_run_redirect',
 454                     'runtime_constraints': {
 455                         'vcpus': 1,
 456                         'ram': 268435456
 457                     },
 458                     'use_existing': True,
 459                     'priority': 500,
 460                     'mounts': {
 461                         '/tmp': {'kind': 'tmp',
 462                                  "capacity": 1073741824 },
 463                         '/var/spool/cwl': {'kind': 'tmp',
 464                                            "capacity": 1073741824 },
 465                         "stderr": {
 466                             "kind": "file",
 467                             "path": "/var/spool/cwl/stderr.txt"
 468                         },
 469                         "stdin": {
 470                             "kind": "collection",
 471                             "path": "file.txt",
 472                             "portable_data_hash": "99999999999999999999999999999996+99"
 473                         },
 474                         "stdout": {
 475                             "kind": "file",
 476                             "path": "/var/spool/cwl/stdout.txt"
 477                         },
 478                     },
 479                     'state': 'Committed',
 480                     "output_name": "Output for step test_run_redirect",
 481                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 482                     'output_path': '/var/spool/cwl',
 483                     'output_ttl': 0,
 484                     'container_image': '99999999999999999999999999999993+99',
 485                     'command': ['ls', '/var/spool/cwl'],
 486                     'cwd': '/var/spool/cwl',
 487                     'scheduling_parameters': {},
 488                     'properties': {},
 489                     'secret_mounts': {},
 490                     'output_storage_classes': ["default"]
 491                 }))
 492
 493     @mock.patch("arvados.collection.Collection")
 494     def test_done(self, col):
 495         api = mock.MagicMock()
 496
 497         runner = mock.MagicMock()
 498         runner.api = api
 499         runner.num_retries = 0
 500         runner.ignore_docker_for_reuse = False
 501         runner.intermediate_output_ttl = 0
 502         runner.secret_store = cwltool.secrets.SecretStore()
 503
 504         runner.api.containers().get().execute.return_value = {"state":"Complete",
 505                                                               "output": "abc+123",
 506                                                               "exit_code": 0}
 507
 508         col().open.return_value = []
 509
 510         loadingContext, runtimeContext = self.helper(runner)
 511
 512         arvjob = arvados_cwl.ArvadosContainer(runner,
 513                                               runtimeContext,
 514                                               mock.MagicMock(),
 515                                               {},
 516                                               None,
 517                                               [],
 518                                               [],
 519                                               "testjob")
 520         arvjob.output_callback = mock.MagicMock()
 521         arvjob.collect_outputs = mock.MagicMock()
 522         arvjob.successCodes = [0]
 523         arvjob.outdir = "/var/spool/cwl"
 524         arvjob.output_ttl = 3600
 525
 526         arvjob.collect_outputs.return_value = {"out": "stuff"}
 527
 528         arvjob.done({
 529             "state": "Final",
 530             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 531             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 532             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 533             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 534             "modified_at": "2017-05-26T12:01:22Z"
 535         })
 536
 537         self.assertFalse(api.collections().create.called)
 538         self.assertFalse(runner.runtime_status_error.called)
 539
 540         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 541         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 542         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 543
 544     # Test to make sure we dont call runtime_status_update if we already did
 545     # some where higher up in the call stack
 546     @mock.patch("arvados_cwl.util.get_current_container")
 547     def test_recursive_runtime_status_update(self, gcc_mock):
 548         self.setup_and_test_container_executor_and_logging(gcc_mock)
 549         root_logger = logging.getLogger('')
 550
 551         # get_current_container is invoked when we call runtime_status_update
 552         # so try and log again!
 553         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 554         try:
 555             root_logger.error("First Error")
 556         except RuntimeError:
 557             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 558
 559
 560     # Test to make sure that an exception raised from
 561     # get_current_container doesn't cause the logger to raise an
 562     # exception
 563     @mock.patch("arvados_cwl.util.get_current_container")
 564     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 565         self.setup_and_test_container_executor_and_logging(gcc_mock)
 566         root_logger = logging.getLogger('')
 567
 568         # get_current_container is invoked when we call
 569         # runtime_status_update, it is going to also raise an
 570         # exception.
 571         gcc_mock.side_effect = Exception("Second Error")
 572         try:
 573             root_logger.error("First Error")
 574         except Exception:
 575             self.fail("Exception in logger should not propagate")
 576         self.assertTrue(gcc_mock.called)
 577
 578     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 579     @mock.patch("arvados_cwl.util.get_current_container")
 580     @mock.patch("arvados.collection.CollectionReader")
 581     @mock.patch("arvados.collection.Collection")
 582     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 583         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 584
 585         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 586         self.assertTrue(gcc_mock.called)
 587
 588         runner.num_retries = 0
 589         runner.ignore_docker_for_reuse = False
 590         runner.intermediate_output_ttl = 0
 591         runner.secret_store = cwltool.secrets.SecretStore()
 592         runner.label = mock.MagicMock()
 593         runner.label.return_value = '[container testjob]'
 594
 595         runner.api.containers().get().execute.return_value = {
 596             "state":"Complete",
 597             "output": "abc+123",
 598             "exit_code": 1,
 599             "log": "def+234"
 600         }
 601
 602         col().open.return_value = []
 603
 604         loadingContext, runtimeContext = self.helper(runner)
 605
 606         arvjob = arvados_cwl.ArvadosContainer(runner,
 607                                               runtimeContext,
 608                                               mock.MagicMock(),
 609                                               {},
 610                                               None,
 611                                               [],
 612                                               [],
 613                                               "testjob")
 614         arvjob.output_callback = mock.MagicMock()
 615         arvjob.collect_outputs = mock.MagicMock()
 616         arvjob.successCodes = [0]
 617         arvjob.outdir = "/var/spool/cwl"
 618         arvjob.output_ttl = 3600
 619         arvjob.collect_outputs.return_value = {"out": "stuff"}
 620
 621         arvjob.done({
 622             "state": "Final",
 623             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 624             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 625             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 626             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 627             "modified_at": "2017-05-26T12:01:22Z"
 628         })
 629
 630         rts_mock.assert_called_with(
 631             'error',
 632             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 633             '  ** log is empty **'
 634         )
 635         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 636
 637     # The test passes no builder.resources
 638     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 639     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 640     def test_mounts(self, keepdocker):
 641         runner = mock.MagicMock()
 642         runner.ignore_docker_for_reuse = False
 643         runner.intermediate_output_ttl = 0
 644         runner.secret_store = cwltool.secrets.SecretStore()
 645         runner.api._rootDesc = {"revision": "20210628"}
 646
 647         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 648         runner.api.collections().get().execute.return_value = {
 649             "portable_data_hash": "99999999999999999999999999999994+99",
 650             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 651
 652         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 653
 654         tool = cmap({
 655             "inputs": [
 656                 {"id": "p1",
 657                  "type": "Directory"}
 658             ],
 659             "outputs": [],
 660             "baseCommand": "ls",
 661             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 662             "id": "",
 663             "class": "CommandLineTool",
 664             "cwlVersion": "v1.2"
 665         })
 666
 667         loadingContext, runtimeContext = self.helper(runner)
 668         runtimeContext.name = "test_run_mounts"
 669
 670         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 671         arvtool.formatgraph = None
 672         job_order = {
 673             "p1": {
 674                 "class": "Directory",
 675                 "location": "keep:99999999999999999999999999999994+44",
 676                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 677                 "listing": [
 678                     {
 679                         "class": "File",
 680                         "location": "keep:99999999999999999999999999999994+44/file1",
 681                     },
 682                     {
 683                         "class": "File",
 684                         "location": "keep:99999999999999999999999999999994+44/file2",
 685                     }
 686                 ]
 687             }
 688         }
 689         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 690             j.run(runtimeContext)
 691             runner.api.container_requests().create.assert_called_with(
 692                 body=JsonDiffMatcher({
 693                     'environment': {
 694                         'HOME': '/var/spool/cwl',
 695                         'TMPDIR': '/tmp'
 696                     },
 697                     'name': 'test_run_mounts',
 698                     'runtime_constraints': {
 699                         'vcpus': 1,
 700                         'ram': 268435456
 701                     },
 702                     'use_existing': True,
 703                     'priority': 500,
 704                     'mounts': {
 705                         "/keep/99999999999999999999999999999994+44": {
 706                             "kind": "collection",
 707                             "portable_data_hash": "99999999999999999999999999999994+44",
 708                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 709                         },
 710                         '/tmp': {'kind': 'tmp',
 711                                  "capacity": 1073741824 },
 712                         '/var/spool/cwl': {'kind': 'tmp',
 713                                            "capacity": 1073741824 }
 714                     },
 715                     'state': 'Committed',
 716                     'output_name': 'Output for step test_run_mounts',
 717                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 718                     'output_path': '/var/spool/cwl',
 719                     'output_ttl': 0,
 720                     'container_image': '99999999999999999999999999999994+99',
 721                     'command': ['ls', '/var/spool/cwl'],
 722                     'cwd': '/var/spool/cwl',
 723                     'scheduling_parameters': {},
 724                     'properties': {},
 725                     'secret_mounts': {},
 726                     'output_storage_classes': ["default"]
 727                 }))
 728
 729     # The test passes no builder.resources
 730     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 731     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 732     def test_secrets(self, keepdocker):
 733         arvados_cwl.add_arv_hints()
 734         runner = mock.MagicMock()
 735         runner.ignore_docker_for_reuse = False
 736         runner.intermediate_output_ttl = 0
 737         runner.secret_store = cwltool.secrets.SecretStore()
 738         runner.api._rootDesc = {"revision": "20210628"}
 739
 740         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 741         runner.api.collections().get().execute.return_value = {
 742             "portable_data_hash": "99999999999999999999999999999993+99"}
 743
 744         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 745
 746         tool = cmap({"arguments": ["md5sum", "example.conf"],
 747                      "class": "CommandLineTool",
 748                      "cwlVersion": "v1.2",
 749                      "hints": [
 750                          {
 751                              "class": "http://commonwl.org/cwltool#Secrets",
 752                              "secrets": [
 753                                  "#secret_job.cwl/pw"
 754                              ]
 755                          }
 756                      ],
 757                      "id": "",
 758                      "inputs": [
 759                          {
 760                              "id": "#secret_job.cwl/pw",
 761                              "type": "string"
 762                          }
 763                      ],
 764                      "outputs": [
 765                      ],
 766                      "requirements": [
 767                          {
 768                              "class": "InitialWorkDirRequirement",
 769                              "listing": [
 770                                  {
 771                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 772                                      "entryname": "example.conf"
 773                                  }
 774                              ]
 775                          }
 776                      ]})
 777
 778         loadingContext, runtimeContext = self.helper(runner)
 779         runtimeContext.name = "test_secrets"
 780
 781         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 782         arvtool.formatgraph = None
 783
 784         job_order = {"pw": "blorp"}
 785         runner.secret_store.store(["pw"], job_order)
 786
 787         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 788             j.run(runtimeContext)
 789             runner.api.container_requests().create.assert_called_with(
 790                 body=JsonDiffMatcher({
 791                     'environment': {
 792                         'HOME': '/var/spool/cwl',
 793                         'TMPDIR': '/tmp'
 794                     },
 795                     'name': 'test_secrets',
 796                     'runtime_constraints': {
 797                         'vcpus': 1,
 798                         'ram': 268435456
 799                     },
 800                     'use_existing': True,
 801                     'priority': 500,
 802                     'mounts': {
 803                         '/tmp': {'kind': 'tmp',
 804                                  "capacity": 1073741824
 805                              },
 806                         '/var/spool/cwl': {'kind': 'tmp',
 807                                            "capacity": 1073741824 }
 808                     },
 809                     'state': 'Committed',
 810                     'output_name': 'Output for step test_secrets',
 811                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 812                     'output_path': '/var/spool/cwl',
 813                     'output_ttl': 0,
 814                     'container_image': '99999999999999999999999999999993+99',
 815                     'command': ['md5sum', 'example.conf'],
 816                     'cwd': '/var/spool/cwl',
 817                     'scheduling_parameters': {},
 818                     'properties': {},
 819                     "secret_mounts": {
 820                         "/var/spool/cwl/example.conf": {
 821                             "content": "username: user\npassword: blorp\n",
 822                             "kind": "text"
 823                         }
 824                     },
 825                     'output_storage_classes': ["default"]
 826                 }))
 827
 828     # The test passes no builder.resources
 829     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 830     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 831     def test_timelimit(self, keepdocker):
 832         runner = mock.MagicMock()
 833         runner.ignore_docker_for_reuse = False
 834         runner.intermediate_output_ttl = 0
 835         runner.secret_store = cwltool.secrets.SecretStore()
 836         runner.api._rootDesc = {"revision": "20210628"}
 837
 838         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 839         runner.api.collections().get().execute.return_value = {
 840             "portable_data_hash": "99999999999999999999999999999993+99"}
 841
 842         tool = cmap({
 843             "inputs": [],
 844             "outputs": [],
 845             "baseCommand": "ls",
 846             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 847             "id": "",
 848             "cwlVersion": "v1.2",
 849             "class": "CommandLineTool",
 850             "hints": [
 851                 {
 852                     "class": "ToolTimeLimit",
 853                     "timelimit": 42
 854                 }
 855             ]
 856         })
 857
 858         loadingContext, runtimeContext = self.helper(runner)
 859         runtimeContext.name = "test_timelimit"
 860
 861         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 862         arvtool.formatgraph = None
 863
 864         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 865             j.run(runtimeContext)
 866
 867         _, kwargs = runner.api.container_requests().create.call_args
 868         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 869
 870
 871     # The test passes no builder.resources
 872     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 873     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 874     def test_setting_storage_class(self, keepdocker):
 875         arv_docker_clear_cache()
 876
 877         runner = mock.MagicMock()
 878         runner.ignore_docker_for_reuse = False
 879         runner.intermediate_output_ttl = 0
 880         runner.secret_store = cwltool.secrets.SecretStore()
 881         runner.api._rootDesc = {"revision": "20210628"}
 882
 883         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 884         runner.api.collections().get().execute.return_value = {
 885             "portable_data_hash": "99999999999999999999999999999993+99"}
 886
 887         tool = cmap({
 888             "inputs": [],
 889             "outputs": [],
 890             "baseCommand": "ls",
 891             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 892             "id": "",
 893             "cwlVersion": "v1.2",
 894             "class": "CommandLineTool",
 895             "hints": [
 896                 {
 897                     "class": "http://arvados.org/cwl#OutputStorageClass",
 898                     "finalStorageClass": ["baz_sc", "qux_sc"],
 899                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 900                 }
 901             ]
 902         })
 903
 904         loadingContext, runtimeContext = self.helper(runner, True)
 905
 906         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 907         arvtool.formatgraph = None
 908
 909         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 910             j.run(runtimeContext)
 911             runner.api.container_requests().create.assert_called_with(
 912                 body=JsonDiffMatcher({
 913                     'environment': {
 914                         'HOME': '/var/spool/cwl',
 915                         'TMPDIR': '/tmp'
 916                     },
 917                     'name': 'test_run_True',
 918                     'runtime_constraints': {
 919                         'vcpus': 1,
 920                         'ram': 268435456
 921                     },
 922                     'use_existing': True,
 923                     'priority': 500,
 924                     'mounts': {
 925                         '/tmp': {'kind': 'tmp',
 926                                  "capacity": 1073741824
 927                              },
 928                         '/var/spool/cwl': {'kind': 'tmp',
 929                                            "capacity": 1073741824 }
 930                     },
 931                     'state': 'Committed',
 932                     'output_name': 'Output for step test_run_True',
 933                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 934                     'output_path': '/var/spool/cwl',
 935                     'output_ttl': 0,
 936                     'container_image': '99999999999999999999999999999993+99',
 937                     'command': ['ls', '/var/spool/cwl'],
 938                     'cwd': '/var/spool/cwl',
 939                     'scheduling_parameters': {},
 940                     'properties': {},
 941                     'secret_mounts': {},
 942                     'output_storage_classes': ["foo_sc", "bar_sc"]
 943                 }))
 944
 945
 946     # The test passes no builder.resources
 947     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 948     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 949     def test_setting_process_properties(self, keepdocker):
 950         arv_docker_clear_cache()
 951
 952         runner = mock.MagicMock()
 953         runner.ignore_docker_for_reuse = False
 954         runner.intermediate_output_ttl = 0
 955         runner.secret_store = cwltool.secrets.SecretStore()
 956         runner.api._rootDesc = {"revision": "20210628"}
 957
 958         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 959         runner.api.collections().get().execute.return_value = {
 960             "portable_data_hash": "99999999999999999999999999999993+99"}
 961
 962         tool = cmap({
 963             "inputs": [
 964                 {"id": "x", "type": "string"}],
 965             "outputs": [],
 966             "baseCommand": "ls",
 967             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 968             "id": "",
 969             "class": "CommandLineTool",
 970             "cwlVersion": "v1.2",
 971             "hints": [
 972             {
 973                 "class": "http://arvados.org/cwl#ProcessProperties",
 974                 "processProperties": [
 975                     {"propertyName": "foo",
 976                      "propertyValue": "bar"},
 977                     {"propertyName": "baz",
 978                      "propertyValue": "$(inputs.x)"},
 979                     {"propertyName": "quux",
 980                      "propertyValue": {
 981                          "q1": 1,
 982                          "q2": 2
 983                      }
 984                     }
 985                 ],
 986             }
 987         ]
 988         })
 989
 990         loadingContext, runtimeContext = self.helper(runner, True)
 991
 992         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 993         arvtool.formatgraph = None
 994
 995         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 996             j.run(runtimeContext)
 997             runner.api.container_requests().create.assert_called_with(
 998                 body=JsonDiffMatcher({
 999                     'environment': {
1000                         'HOME': '/var/spool/cwl',
1001                         'TMPDIR': '/tmp'
1002                     },
1003                     'name': 'test_run_True',
1004                     'runtime_constraints': {
1005                         'vcpus': 1,
1006                         'ram': 268435456
1007                     },
1008                     'use_existing': True,
1009                     'priority': 500,
1010                     'mounts': {
1011                         '/tmp': {'kind': 'tmp',
1012                                  "capacity": 1073741824
1013                              },
1014                         '/var/spool/cwl': {'kind': 'tmp',
1015                                            "capacity": 1073741824 }
1016                     },
1017                     'state': 'Committed',
1018                     'output_name': 'Output for step test_run_True',
1019                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1020                     'output_path': '/var/spool/cwl',
1021                     'output_ttl': 0,
1022                     'container_image': '99999999999999999999999999999993+99',
1023                     'command': ['ls', '/var/spool/cwl'],
1024                     'cwd': '/var/spool/cwl',
1025                     'scheduling_parameters': {},
1026                     'properties': {
1027                         "baz": "blorp",
1028                         "foo": "bar",
1029                         "quux": {
1030                             "q1": 1,
1031                             "q2": 2
1032                         }
1033                     },
1034                     'secret_mounts': {},
1035                     'output_storage_classes': ["default"]
1036                 }))
1037
1038
1039     # The test passes no builder.resources
1040     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1041     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1042     def test_cuda_requirement(self, keepdocker):
1043         arvados_cwl.add_arv_hints()
1044         arv_docker_clear_cache()
1045
1046         runner = mock.MagicMock()
1047         runner.ignore_docker_for_reuse = False
1048         runner.intermediate_output_ttl = 0
1049         runner.secret_store = cwltool.secrets.SecretStore()
1050         runner.api._rootDesc = {"revision": "20210628"}
1051
1052         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1053         runner.api.collections().get().execute.return_value = {
1054             "portable_data_hash": "99999999999999999999999999999993+99"}
1055
1056         tool = cmap({
1057             "inputs": [],
1058             "outputs": [],
1059             "baseCommand": "nvidia-smi",
1060             "arguments": [],
1061             "id": "",
1062             "cwlVersion": "v1.2",
1063             "class": "CommandLineTool",
1064             "requirements": [
1065             {
1066                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1067                 "cudaVersionMin": "11.0",
1068                 "cudaComputeCapabilityMin": "9.0",
1069             }
1070         ]
1071         })
1072
1073         loadingContext, runtimeContext = self.helper(runner, True)
1074
1075         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1076         arvtool.formatgraph = None
1077
1078         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1079             j.run(runtimeContext)
1080             runner.api.container_requests().create.assert_called_with(
1081                 body=JsonDiffMatcher({
1082                     'environment': {
1083                         'HOME': '/var/spool/cwl',
1084                         'TMPDIR': '/tmp'
1085                     },
1086                     'name': 'test_run_True',
1087                     'runtime_constraints': {
1088                         'vcpus': 1,
1089                         'ram': 268435456,
1090                         'cuda': {
1091                             'device_count': 1,
1092                             'driver_version': "11.0",
1093                             'hardware_capability': "9.0"
1094                         }
1095                     },
1096                     'use_existing': True,
1097                     'priority': 500,
1098                     'mounts': {
1099                         '/tmp': {'kind': 'tmp',
1100                                  "capacity": 1073741824
1101                              },
1102                         '/var/spool/cwl': {'kind': 'tmp',
1103                                            "capacity": 1073741824 }
1104                     },
1105                     'state': 'Committed',
1106                     'output_name': 'Output for step test_run_True',
1107                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1108                     'output_path': '/var/spool/cwl',
1109                     'output_ttl': 0,
1110                     'container_image': '99999999999999999999999999999993+99',
1111                     'command': ['nvidia-smi'],
1112                     'cwd': '/var/spool/cwl',
1113                     'scheduling_parameters': {},
1114                     'properties': {},
1115                     'secret_mounts': {},
1116                     'output_storage_classes': ["default"]
1117                 }))
1118
1119
1120 class TestWorkflow(unittest.TestCase):
1121     def setUp(self):
1122         cwltool.process._names = set()
1123         arv_docker_clear_cache()
1124
1125     def helper(self, runner, enable_reuse=True):
1126         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1127
1128         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1129                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1130
1131         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1132         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1133         document_loader.fetch_text = document_loader.fetcher.fetch_text
1134         document_loader.check_exists = document_loader.fetcher.check_exists
1135
1136         loadingContext = arvados_cwl.context.ArvLoadingContext(
1137             {"avsc_names": avsc_names,
1138              "basedir": "",
1139              "make_fs_access": make_fs_access,
1140              "loader": document_loader,
1141              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1142              "construct_tool_object": runner.arv_make_tool})
1143         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1144             {"work_api": "containers",
1145              "basedir": "",
1146              "name": "test_run_wf_"+str(enable_reuse),
1147              "make_fs_access": make_fs_access,
1148              "tmpdir": "/tmp",
1149              "enable_reuse": enable_reuse,
1150              "priority": 500})
1151
1152         return loadingContext, runtimeContext
1153
1154     # The test passes no builder.resources
1155     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1156     @mock.patch("arvados.collection.CollectionReader")
1157     @mock.patch("arvados.collection.Collection")
1158     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1159     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1160         arvados_cwl.add_arv_hints()
1161
1162         api = mock.MagicMock()
1163         api._rootDesc = get_rootDesc()
1164
1165         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1166         self.assertEqual(runner.work_api, 'containers')
1167
1168         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1169         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1170         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1171                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1172
1173         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1174         runner.ignore_docker_for_reuse = False
1175         runner.num_retries = 0
1176         runner.secret_store = cwltool.secrets.SecretStore()
1177
1178         loadingContext, runtimeContext = self.helper(runner)
1179         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1180
1181         mockcollectionreader().exists.return_value = True
1182
1183         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1184         metadata["cwlVersion"] = tool["cwlVersion"]
1185
1186         mockc = mock.MagicMock()
1187         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1188         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1189
1190         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1191         arvtool.formatgraph = None
1192         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1193
1194         next(it).run(runtimeContext)
1195         next(it).run(runtimeContext)
1196
1197         with open("tests/wf/scatter2_subwf.cwl") as f:
1198             subwf = StripYAMLComments(f.read()).rstrip()
1199
1200         runner.api.container_requests().create.assert_called_with(
1201             body=JsonDiffMatcher({
1202                 "command": [
1203                     "cwltool",
1204                     "--no-container",
1205                     "--move-outputs",
1206                     "--preserve-entire-environment",
1207                     "workflow.cwl",
1208                     "cwl.input.yml"
1209                 ],
1210                 "container_image": "99999999999999999999999999999993+99",
1211                 "cwd": "/var/spool/cwl",
1212                 "environment": {
1213                     "HOME": "/var/spool/cwl",
1214                     "TMPDIR": "/tmp"
1215                 },
1216                 "mounts": {
1217                     "/keep/99999999999999999999999999999999+118": {
1218                         "kind": "collection",
1219                         "portable_data_hash": "99999999999999999999999999999999+118"
1220                     },
1221                     "/tmp": {
1222                         "capacity": 1073741824,
1223                         "kind": "tmp"
1224                     },
1225                     "/var/spool/cwl": {
1226                         "capacity": 1073741824,
1227                         "kind": "tmp"
1228                     },
1229                     "/var/spool/cwl/cwl.input.yml": {
1230                         "kind": "collection",
1231                         "path": "cwl.input.yml",
1232                         "portable_data_hash": "99999999999999999999999999999996+99"
1233                     },
1234                     "/var/spool/cwl/workflow.cwl": {
1235                         "kind": "collection",
1236                         "path": "workflow.cwl",
1237                         "portable_data_hash": "99999999999999999999999999999996+99"
1238                     },
1239                     "stdout": {
1240                         "kind": "file",
1241                         "path": "/var/spool/cwl/cwl.output.json"
1242                     }
1243                 },
1244                 "name": "scatterstep",
1245                 "output_name": "Output for step scatterstep",
1246                 "output_path": "/var/spool/cwl",
1247                 "output_ttl": 0,
1248                 "priority": 500,
1249                 "properties": {},
1250                 "runtime_constraints": {
1251                     "ram": 1073741824,
1252                     "vcpus": 1
1253                 },
1254                 "scheduling_parameters": {},
1255                 "secret_mounts": {},
1256                 "state": "Committed",
1257                 "use_existing": True,
1258                 'output_storage_classes': ["default"]
1259             }))
1260         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1261         mockc.open().__enter__().write.assert_has_calls([mock.call(
1262 '''{
1263   "fileblub": {
1264     "basename": "token.txt",
1265     "class": "File",
1266     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1267     "size": 0
1268   },
1269   "sleeptime": 5
1270 }''')])
1271
1272     # The test passes no builder.resources
1273     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1274     @mock.patch("arvados.collection.CollectionReader")
1275     @mock.patch("arvados.collection.Collection")
1276     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1277     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1278         arvados_cwl.add_arv_hints()
1279
1280         api = mock.MagicMock()
1281         api._rootDesc = get_rootDesc()
1282
1283         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1284         self.assertEqual(runner.work_api, 'containers')
1285
1286         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1287         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1288                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1289         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1290                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1291
1292         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1293         runner.ignore_docker_for_reuse = False
1294         runner.num_retries = 0
1295         runner.secret_store = cwltool.secrets.SecretStore()
1296
1297         loadingContext, runtimeContext = self.helper(runner)
1298         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1299         loadingContext.do_update = True
1300         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1301
1302         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1303
1304         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1305         arvtool.formatgraph = None
1306         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1307
1308         next(it).run(runtimeContext)
1309         next(it).run(runtimeContext)
1310
1311         with open("tests/wf/echo-subwf.cwl") as f:
1312             subwf = StripYAMLComments(f.read())
1313
1314         runner.api.container_requests().create.assert_called_with(
1315             body=JsonDiffMatcher({
1316                 'output_ttl': 0,
1317                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1318                 'scheduling_parameters': {},
1319                 'name': u'echo-subwf',
1320                 'secret_mounts': {},
1321                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1322                 'properties': {},
1323                 'priority': 500,
1324                 'mounts': {
1325                     '/var/spool/cwl/cwl.input.yml': {
1326                         'portable_data_hash': '99999999999999999999999999999996+99',
1327                         'kind': 'collection',
1328                         'path': 'cwl.input.yml'
1329                     },
1330                     '/var/spool/cwl/workflow.cwl': {
1331                         'portable_data_hash': '99999999999999999999999999999996+99',
1332                         'kind': 'collection',
1333                         'path': 'workflow.cwl'
1334                     },
1335                     'stdout': {
1336                         'path': '/var/spool/cwl/cwl.output.json',
1337                         'kind': 'file'
1338                     },
1339                     '/tmp': {
1340                         'kind': 'tmp',
1341                         'capacity': 1073741824
1342                     }, '/var/spool/cwl': {
1343                         'kind': 'tmp',
1344                         'capacity': 3221225472
1345                     }
1346                 },
1347                 'state': 'Committed',
1348                 'output_path': '/var/spool/cwl',
1349                 'container_image': '99999999999999999999999999999993+99',
1350                 'command': [
1351                     u'cwltool',
1352                     u'--no-container',
1353                     u'--move-outputs',
1354                     u'--preserve-entire-environment',
1355                     u'workflow.cwl',
1356                     u'cwl.input.yml'
1357                 ],
1358                 'use_existing': True,
1359                 'output_name': u'Output for step echo-subwf',
1360                 'cwd': '/var/spool/cwl',
1361                 'output_storage_classes': ["default"]
1362             }))
1363
1364     def test_default_work_api(self):
1365         arvados_cwl.add_arv_hints()
1366
1367         api = mock.MagicMock()
1368         api._rootDesc = copy.deepcopy(get_rootDesc())
1369         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1370         self.assertEqual(runner.work_api, 'containers')