sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def helper(self, runner, enable_reuse=True):
  67         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  68
  69         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  70                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  71         fs_access = mock.MagicMock()
  72         fs_access.exists.return_value = True
  73
  74         loadingContext = arvados_cwl.context.ArvLoadingContext(
  75             {"avsc_names": avsc_names,
  76              "basedir": "",
  77              "make_fs_access": make_fs_access,
  78              "construct_tool_object": runner.arv_make_tool,
  79              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  80              "loader": Loader({}),
  81              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  82              })
  83         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  84             {"work_api": "containers",
  85              "basedir": "",
  86              "name": "test_run_"+str(enable_reuse),
  87              "make_fs_access": make_fs_access,
  88              "tmpdir": "/tmp",
  89              "outdir": "/tmp",
  90              "enable_reuse": enable_reuse,
  91              "priority": 500,
  92              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
  93              "workflow_eval_lock": threading.Condition(threading.RLock())
  94             })
  95
  96         if isinstance(runner, mock.MagicMock):
  97             def make_tool(toolpath_object, loadingContext):
  98                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
  99             runner.arv_make_tool.side_effect = make_tool
 100
 101         return loadingContext, runtimeContext
 102
 103     # Helper function to set up the ArvCwlExecutor to use the containers api
 104     # and test that the RuntimeStatusLoggingHandler is set up correctly
 105     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 106         api = mock.MagicMock()
 107         api._rootDesc = copy.deepcopy(get_rootDesc())
 108
 109         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 110         # adds the logging handler that will call runtime_status_update() mock
 111         self.assertFalse(gcc_mock.called)
 112         runner = arvados_cwl.ArvCwlExecutor(api)
 113         self.assertEqual(runner.work_api, 'containers')
 114         root_logger = logging.getLogger('')
 115         handlerClasses = [h.__class__ for h in root_logger.handlers]
 116         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 117         return runner
 118
 119     # The test passes no builder.resources
 120     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 121     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 122     def test_run(self, keepdocker):
 123         for enable_reuse in (True, False):
 124             arv_docker_clear_cache()
 125
 126             runner = mock.MagicMock()
 127             runner.ignore_docker_for_reuse = False
 128             runner.intermediate_output_ttl = 0
 129             runner.secret_store = cwltool.secrets.SecretStore()
 130             runner.api._rootDesc = {"revision": "20210628"}
 131
 132             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 133             runner.api.collections().get().execute.return_value = {
 134                 "portable_data_hash": "99999999999999999999999999999993+99"}
 135
 136             tool = cmap({
 137                 "inputs": [],
 138                 "outputs": [],
 139                 "baseCommand": "ls",
 140                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 141                 "id": "",
 142                 "class": "CommandLineTool",
 143                 "cwlVersion": "v1.2"
 144             })
 145
 146             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 147
 148             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 149             arvtool.formatgraph = None
 150
 151             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 152                 j.run(runtimeContext)
 153                 runner.api.container_requests().create.assert_called_with(
 154                     body=JsonDiffMatcher({
 155                         'environment': {
 156                             'HOME': '/var/spool/cwl',
 157                             'TMPDIR': '/tmp'
 158                         },
 159                         'name': 'test_run_'+str(enable_reuse),
 160                         'runtime_constraints': {
 161                             'vcpus': 1,
 162                             'ram': 268435456
 163                         },
 164                         'use_existing': enable_reuse,
 165                         'priority': 500,
 166                         'mounts': {
 167                             '/tmp': {'kind': 'tmp',
 168                                      "capacity": 1073741824
 169                                  },
 170                             '/var/spool/cwl': {'kind': 'tmp',
 171                                                "capacity": 1073741824 }
 172                         },
 173                         'state': 'Committed',
 174                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 175                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 176                         'output_path': '/var/spool/cwl',
 177                         'output_ttl': 0,
 178                         'container_image': '99999999999999999999999999999993+99',
 179                         'command': ['ls', '/var/spool/cwl'],
 180                         'cwd': '/var/spool/cwl',
 181                         'scheduling_parameters': {},
 182                         'properties': {},
 183                         'secret_mounts': {},
 184                         'output_storage_classes': ["default"]
 185                     }))
 186
 187     # The test passes some fields in builder.resources
 188     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 189     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 190     def test_resource_requirements(self, keepdocker):
 191         arvados_cwl.add_arv_hints()
 192         runner = mock.MagicMock()
 193         runner.ignore_docker_for_reuse = False
 194         runner.intermediate_output_ttl = 3600
 195         runner.secret_store = cwltool.secrets.SecretStore()
 196         runner.api._rootDesc = {"revision": "20210628"}
 197
 198         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 199         runner.api.collections().get().execute.return_value = {
 200             "portable_data_hash": "99999999999999999999999999999993+99"}
 201
 202         tool = cmap({
 203             "inputs": [],
 204             "outputs": [],
 205             "hints": [{
 206                 "class": "ResourceRequirement",
 207                 "coresMin": 3,
 208                 "ramMin": 3000,
 209                 "tmpdirMin": 4000,
 210                 "outdirMin": 5000
 211             }, {
 212                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 213                 "keep_cache": 512
 214             }, {
 215                 "class": "http://arvados.org/cwl#APIRequirement",
 216             }, {
 217                 "class": "http://arvados.org/cwl#PartitionRequirement",
 218                 "partition": "blurb"
 219             }, {
 220                 "class": "http://arvados.org/cwl#IntermediateOutput",
 221                 "outputTTL": 7200
 222             }, {
 223                 "class": "WorkReuse",
 224                 "enableReuse": False
 225             }],
 226             "baseCommand": "ls",
 227             "id": "",
 228             "class": "CommandLineTool",
 229             "cwlVersion": "v1.2"
 230         })
 231
 232         loadingContext, runtimeContext = self.helper(runner)
 233         runtimeContext.name = "test_resource_requirements"
 234
 235         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 236         arvtool.formatgraph = None
 237         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 238             j.run(runtimeContext)
 239
 240         call_args, call_kwargs = runner.api.container_requests().create.call_args
 241
 242         call_body_expected = {
 243             'environment': {
 244                 'HOME': '/var/spool/cwl',
 245                 'TMPDIR': '/tmp'
 246             },
 247             'name': 'test_resource_requirements',
 248             'runtime_constraints': {
 249                 'vcpus': 3,
 250                 'ram': 3145728000,
 251                 'keep_cache_ram': 536870912,
 252                 'API': True
 253             },
 254             'use_existing': False,
 255             'priority': 500,
 256             'mounts': {
 257                 '/tmp': {'kind': 'tmp',
 258                          "capacity": 4194304000 },
 259                 '/var/spool/cwl': {'kind': 'tmp',
 260                                    "capacity": 5242880000 }
 261             },
 262             'state': 'Committed',
 263             'output_name': 'Output for step test_resource_requirements',
 264             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 265             'output_path': '/var/spool/cwl',
 266             'output_ttl': 7200,
 267             'container_image': '99999999999999999999999999999993+99',
 268             'command': ['ls'],
 269             'cwd': '/var/spool/cwl',
 270             'scheduling_parameters': {
 271                 'partitions': ['blurb']
 272             },
 273             'properties': {},
 274             'secret_mounts': {},
 275             'output_storage_classes': ["default"]
 276         }
 277
 278         call_body = call_kwargs.get('body', None)
 279         self.assertNotEqual(None, call_body)
 280         for key in call_body:
 281             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 282
 283
 284     # The test passes some fields in builder.resources
 285     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 286     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 287     @mock.patch("arvados.collection.Collection")
 288     def test_initial_work_dir(self, collection_mock, keepdocker):
 289         runner = mock.MagicMock()
 290         runner.ignore_docker_for_reuse = False
 291         runner.intermediate_output_ttl = 0
 292         runner.secret_store = cwltool.secrets.SecretStore()
 293         runner.api._rootDesc = {"revision": "20210628"}
 294
 295         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 296         runner.api.collections().get().execute.return_value = {
 297             "portable_data_hash": "99999999999999999999999999999993+99"}
 298
 299         sourcemock = mock.MagicMock()
 300         def get_collection_mock(p):
 301             if "/" in p:
 302                 return (sourcemock, p.split("/", 1)[1])
 303             else:
 304                 return (sourcemock, "")
 305         runner.fs_access.get_collection.side_effect = get_collection_mock
 306
 307         vwdmock = mock.MagicMock()
 308         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 309
 310         tool = cmap({
 311             "inputs": [],
 312             "outputs": [],
 313             "hints": [{
 314                 "class": "InitialWorkDirRequirement",
 315                 "listing": [{
 316                     "class": "File",
 317                     "basename": "foo",
 318                     "location": "keep:99999999999999999999999999999995+99/bar"
 319                 },
 320                 {
 321                     "class": "Directory",
 322                     "basename": "foo2",
 323                     "location": "keep:99999999999999999999999999999995+99"
 324                 },
 325                 {
 326                     "class": "File",
 327                     "basename": "filename",
 328                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "subdir",
 333                     "location": "keep:99999999999999999999999999999995+99/subdir"
 334                 }                        ]
 335             }],
 336             "baseCommand": "ls",
 337             "class": "CommandLineTool",
 338             "cwlVersion": "v1.2",
 339             "id": ""
 340         })
 341
 342         loadingContext, runtimeContext = self.helper(runner)
 343         runtimeContext.name = "test_initial_work_dir"
 344
 345         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 346
 347         arvtool.formatgraph = None
 348         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 349             j.run(runtimeContext)
 350
 351         call_args, call_kwargs = runner.api.container_requests().create.call_args
 352
 353         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 354         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 355         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 356         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 357
 358         call_body_expected = {
 359             'environment': {
 360                 'HOME': '/var/spool/cwl',
 361                 'TMPDIR': '/tmp'
 362             },
 363             'name': 'test_initial_work_dir',
 364             'runtime_constraints': {
 365                 'vcpus': 1,
 366                 'ram': 268435456
 367             },
 368             'use_existing': True,
 369             'priority': 500,
 370             'mounts': {
 371                 '/tmp': {'kind': 'tmp',
 372                          "capacity": 1073741824 },
 373                 '/var/spool/cwl': {'kind': 'tmp',
 374                                    "capacity": 1073741824 },
 375                 '/var/spool/cwl/foo': {
 376                     'kind': 'collection',
 377                     'path': 'foo',
 378                     'portable_data_hash': '99999999999999999999999999999996+99'
 379                 },
 380                 '/var/spool/cwl/foo2': {
 381                     'kind': 'collection',
 382                     'path': 'foo2',
 383                     'portable_data_hash': '99999999999999999999999999999996+99'
 384                 },
 385                 '/var/spool/cwl/filename': {
 386                     'kind': 'collection',
 387                     'path': 'filename',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/subdir': {
 391                     'kind': 'collection',
 392                     'path': 'subdir',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 }
 395             },
 396             'state': 'Committed',
 397             'output_name': 'Output for step test_initial_work_dir',
 398             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 399             'output_path': '/var/spool/cwl',
 400             'output_ttl': 0,
 401             'container_image': '99999999999999999999999999999993+99',
 402             'command': ['ls'],
 403             'cwd': '/var/spool/cwl',
 404             'scheduling_parameters': {
 405             },
 406             'properties': {},
 407             'secret_mounts': {},
 408             'output_storage_classes': ["default"]
 409         }
 410
 411         call_body = call_kwargs.get('body', None)
 412         self.assertNotEqual(None, call_body)
 413         for key in call_body:
 414             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 415
 416
 417     # Test redirecting stdin/stdout/stderr
 418     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 419     def test_redirects(self, keepdocker):
 420         runner = mock.MagicMock()
 421         runner.ignore_docker_for_reuse = False
 422         runner.intermediate_output_ttl = 0
 423         runner.secret_store = cwltool.secrets.SecretStore()
 424         runner.api._rootDesc = {"revision": "20210628"}
 425
 426         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 427         runner.api.collections().get().execute.return_value = {
 428             "portable_data_hash": "99999999999999999999999999999993+99"}
 429
 430         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 431
 432         tool = cmap({
 433             "inputs": [],
 434             "outputs": [],
 435             "baseCommand": "ls",
 436             "stdout": "stdout.txt",
 437             "stderr": "stderr.txt",
 438             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 439             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 440             "id": "",
 441             "class": "CommandLineTool",
 442             "cwlVersion": "v1.2"
 443         })
 444
 445         loadingContext, runtimeContext = self.helper(runner)
 446         runtimeContext.name = "test_run_redirect"
 447
 448         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 449         arvtool.formatgraph = None
 450         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 451             j.run(runtimeContext)
 452             runner.api.container_requests().create.assert_called_with(
 453                 body=JsonDiffMatcher({
 454                     'environment': {
 455                         'HOME': '/var/spool/cwl',
 456                         'TMPDIR': '/tmp'
 457                     },
 458                     'name': 'test_run_redirect',
 459                     'runtime_constraints': {
 460                         'vcpus': 1,
 461                         'ram': 268435456
 462                     },
 463                     'use_existing': True,
 464                     'priority': 500,
 465                     'mounts': {
 466                         '/tmp': {'kind': 'tmp',
 467                                  "capacity": 1073741824 },
 468                         '/var/spool/cwl': {'kind': 'tmp',
 469                                            "capacity": 1073741824 },
 470                         "stderr": {
 471                             "kind": "file",
 472                             "path": "/var/spool/cwl/stderr.txt"
 473                         },
 474                         "stdin": {
 475                             "kind": "collection",
 476                             "path": "file.txt",
 477                             "portable_data_hash": "99999999999999999999999999999996+99"
 478                         },
 479                         "stdout": {
 480                             "kind": "file",
 481                             "path": "/var/spool/cwl/stdout.txt"
 482                         },
 483                     },
 484                     'state': 'Committed',
 485                     "output_name": "Output for step test_run_redirect",
 486                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 487                     'output_path': '/var/spool/cwl',
 488                     'output_ttl': 0,
 489                     'container_image': '99999999999999999999999999999993+99',
 490                     'command': ['ls', '/var/spool/cwl'],
 491                     'cwd': '/var/spool/cwl',
 492                     'scheduling_parameters': {},
 493                     'properties': {},
 494                     'secret_mounts': {},
 495                     'output_storage_classes': ["default"]
 496                 }))
 497
 498     @mock.patch("arvados.collection.Collection")
 499     def test_done(self, col):
 500         api = mock.MagicMock()
 501
 502         runner = mock.MagicMock()
 503         runner.api = api
 504         runner.num_retries = 0
 505         runner.ignore_docker_for_reuse = False
 506         runner.intermediate_output_ttl = 0
 507         runner.secret_store = cwltool.secrets.SecretStore()
 508
 509         runner.api.containers().get().execute.return_value = {"state":"Complete",
 510                                                               "output": "abc+123",
 511                                                               "exit_code": 0}
 512
 513         col().open.return_value = []
 514
 515         loadingContext, runtimeContext = self.helper(runner)
 516
 517         arvjob = arvados_cwl.ArvadosContainer(runner,
 518                                               runtimeContext,
 519                                               mock.MagicMock(),
 520                                               {},
 521                                               None,
 522                                               [],
 523                                               [],
 524                                               "testjob")
 525         arvjob.output_callback = mock.MagicMock()
 526         arvjob.collect_outputs = mock.MagicMock()
 527         arvjob.successCodes = [0]
 528         arvjob.outdir = "/var/spool/cwl"
 529         arvjob.output_ttl = 3600
 530
 531         arvjob.collect_outputs.return_value = {"out": "stuff"}
 532
 533         arvjob.done({
 534             "state": "Final",
 535             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 536             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 537             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 538             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 539             "modified_at": "2017-05-26T12:01:22Z"
 540         })
 541
 542         self.assertFalse(api.collections().create.called)
 543         self.assertFalse(runner.runtime_status_error.called)
 544
 545         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 546         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 547         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 548
 549     # Test to make sure we dont call runtime_status_update if we already did
 550     # some where higher up in the call stack
 551     @mock.patch("arvados_cwl.util.get_current_container")
 552     def test_recursive_runtime_status_update(self, gcc_mock):
 553         self.setup_and_test_container_executor_and_logging(gcc_mock)
 554         root_logger = logging.getLogger('')
 555
 556         # get_current_container is invoked when we call runtime_status_update
 557         # so try and log again!
 558         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 559         try:
 560             root_logger.error("First Error")
 561         except RuntimeError:
 562             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 563
 564
 565     # Test to make sure that an exception raised from
 566     # get_current_container doesn't cause the logger to raise an
 567     # exception
 568     @mock.patch("arvados_cwl.util.get_current_container")
 569     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 570         self.setup_and_test_container_executor_and_logging(gcc_mock)
 571         root_logger = logging.getLogger('')
 572
 573         # get_current_container is invoked when we call
 574         # runtime_status_update, it is going to also raise an
 575         # exception.
 576         gcc_mock.side_effect = Exception("Second Error")
 577         try:
 578             root_logger.error("First Error")
 579         except Exception:
 580             self.fail("Exception in logger should not propagate")
 581         self.assertTrue(gcc_mock.called)
 582
 583     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 584     @mock.patch("arvados_cwl.util.get_current_container")
 585     @mock.patch("arvados.collection.CollectionReader")
 586     @mock.patch("arvados.collection.Collection")
 587     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 588         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 589
 590         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 591         self.assertTrue(gcc_mock.called)
 592
 593         runner.num_retries = 0
 594         runner.ignore_docker_for_reuse = False
 595         runner.intermediate_output_ttl = 0
 596         runner.secret_store = cwltool.secrets.SecretStore()
 597         runner.label = mock.MagicMock()
 598         runner.label.return_value = '[container testjob]'
 599
 600         runner.api.containers().get().execute.return_value = {
 601             "state":"Complete",
 602             "output": "abc+123",
 603             "exit_code": 1,
 604             "log": "def+234"
 605         }
 606
 607         col().open.return_value = []
 608
 609         loadingContext, runtimeContext = self.helper(runner)
 610
 611         arvjob = arvados_cwl.ArvadosContainer(runner,
 612                                               runtimeContext,
 613                                               mock.MagicMock(),
 614                                               {},
 615                                               None,
 616                                               [],
 617                                               [],
 618                                               "testjob")
 619         arvjob.output_callback = mock.MagicMock()
 620         arvjob.collect_outputs = mock.MagicMock()
 621         arvjob.successCodes = [0]
 622         arvjob.outdir = "/var/spool/cwl"
 623         arvjob.output_ttl = 3600
 624         arvjob.collect_outputs.return_value = {"out": "stuff"}
 625
 626         arvjob.done({
 627             "state": "Final",
 628             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 629             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 630             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 631             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 632             "modified_at": "2017-05-26T12:01:22Z"
 633         })
 634
 635         rts_mock.assert_called_with(
 636             'error',
 637             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 638             '  ** log is empty **'
 639         )
 640         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 641
 642     # The test passes no builder.resources
 643     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 644     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 645     def test_mounts(self, keepdocker):
 646         runner = mock.MagicMock()
 647         runner.ignore_docker_for_reuse = False
 648         runner.intermediate_output_ttl = 0
 649         runner.secret_store = cwltool.secrets.SecretStore()
 650         runner.api._rootDesc = {"revision": "20210628"}
 651
 652         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 653         runner.api.collections().get().execute.return_value = {
 654             "portable_data_hash": "99999999999999999999999999999994+99",
 655             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 656
 657         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 658
 659         tool = cmap({
 660             "inputs": [
 661                 {"id": "p1",
 662                  "type": "Directory"}
 663             ],
 664             "outputs": [],
 665             "baseCommand": "ls",
 666             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 667             "id": "",
 668             "class": "CommandLineTool",
 669             "cwlVersion": "v1.2"
 670         })
 671
 672         loadingContext, runtimeContext = self.helper(runner)
 673         runtimeContext.name = "test_run_mounts"
 674
 675         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 676         arvtool.formatgraph = None
 677         job_order = {
 678             "p1": {
 679                 "class": "Directory",
 680                 "location": "keep:99999999999999999999999999999994+44",
 681                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 682                 "listing": [
 683                     {
 684                         "class": "File",
 685                         "location": "keep:99999999999999999999999999999994+44/file1",
 686                     },
 687                     {
 688                         "class": "File",
 689                         "location": "keep:99999999999999999999999999999994+44/file2",
 690                     }
 691                 ]
 692             }
 693         }
 694         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 695             j.run(runtimeContext)
 696             runner.api.container_requests().create.assert_called_with(
 697                 body=JsonDiffMatcher({
 698                     'environment': {
 699                         'HOME': '/var/spool/cwl',
 700                         'TMPDIR': '/tmp'
 701                     },
 702                     'name': 'test_run_mounts',
 703                     'runtime_constraints': {
 704                         'vcpus': 1,
 705                         'ram': 268435456
 706                     },
 707                     'use_existing': True,
 708                     'priority': 500,
 709                     'mounts': {
 710                         "/keep/99999999999999999999999999999994+44": {
 711                             "kind": "collection",
 712                             "portable_data_hash": "99999999999999999999999999999994+44",
 713                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 714                         },
 715                         '/tmp': {'kind': 'tmp',
 716                                  "capacity": 1073741824 },
 717                         '/var/spool/cwl': {'kind': 'tmp',
 718                                            "capacity": 1073741824 }
 719                     },
 720                     'state': 'Committed',
 721                     'output_name': 'Output for step test_run_mounts',
 722                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 723                     'output_path': '/var/spool/cwl',
 724                     'output_ttl': 0,
 725                     'container_image': '99999999999999999999999999999994+99',
 726                     'command': ['ls', '/var/spool/cwl'],
 727                     'cwd': '/var/spool/cwl',
 728                     'scheduling_parameters': {},
 729                     'properties': {},
 730                     'secret_mounts': {},
 731                     'output_storage_classes': ["default"]
 732                 }))
 733
 734     # The test passes no builder.resources
 735     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 736     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 737     def test_secrets(self, keepdocker):
 738         arvados_cwl.add_arv_hints()
 739         runner = mock.MagicMock()
 740         runner.ignore_docker_for_reuse = False
 741         runner.intermediate_output_ttl = 0
 742         runner.secret_store = cwltool.secrets.SecretStore()
 743         runner.api._rootDesc = {"revision": "20210628"}
 744
 745         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 746         runner.api.collections().get().execute.return_value = {
 747             "portable_data_hash": "99999999999999999999999999999993+99"}
 748
 749         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 750
 751         tool = cmap({"arguments": ["md5sum", "example.conf"],
 752                      "class": "CommandLineTool",
 753                      "cwlVersion": "v1.2",
 754                      "hints": [
 755                          {
 756                              "class": "http://commonwl.org/cwltool#Secrets",
 757                              "secrets": [
 758                                  "#secret_job.cwl/pw"
 759                              ]
 760                          }
 761                      ],
 762                      "id": "",
 763                      "inputs": [
 764                          {
 765                              "id": "#secret_job.cwl/pw",
 766                              "type": "string"
 767                          }
 768                      ],
 769                      "outputs": [
 770                      ],
 771                      "requirements": [
 772                          {
 773                              "class": "InitialWorkDirRequirement",
 774                              "listing": [
 775                                  {
 776                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 777                                      "entryname": "example.conf"
 778                                  }
 779                              ]
 780                          }
 781                      ]})
 782
 783         loadingContext, runtimeContext = self.helper(runner)
 784         runtimeContext.name = "test_secrets"
 785
 786         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 787         arvtool.formatgraph = None
 788
 789         job_order = {"pw": "blorp"}
 790         runner.secret_store.store(["pw"], job_order)
 791
 792         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 793             j.run(runtimeContext)
 794             runner.api.container_requests().create.assert_called_with(
 795                 body=JsonDiffMatcher({
 796                     'environment': {
 797                         'HOME': '/var/spool/cwl',
 798                         'TMPDIR': '/tmp'
 799                     },
 800                     'name': 'test_secrets',
 801                     'runtime_constraints': {
 802                         'vcpus': 1,
 803                         'ram': 268435456
 804                     },
 805                     'use_existing': True,
 806                     'priority': 500,
 807                     'mounts': {
 808                         '/tmp': {'kind': 'tmp',
 809                                  "capacity": 1073741824
 810                              },
 811                         '/var/spool/cwl': {'kind': 'tmp',
 812                                            "capacity": 1073741824 }
 813                     },
 814                     'state': 'Committed',
 815                     'output_name': 'Output for step test_secrets',
 816                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 817                     'output_path': '/var/spool/cwl',
 818                     'output_ttl': 0,
 819                     'container_image': '99999999999999999999999999999993+99',
 820                     'command': ['md5sum', 'example.conf'],
 821                     'cwd': '/var/spool/cwl',
 822                     'scheduling_parameters': {},
 823                     'properties': {},
 824                     "secret_mounts": {
 825                         "/var/spool/cwl/example.conf": {
 826                             "content": "username: user\npassword: blorp\n",
 827                             "kind": "text"
 828                         }
 829                     },
 830                     'output_storage_classes': ["default"]
 831                 }))
 832
 833     # The test passes no builder.resources
 834     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 835     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 836     def test_timelimit(self, keepdocker):
 837         runner = mock.MagicMock()
 838         runner.ignore_docker_for_reuse = False
 839         runner.intermediate_output_ttl = 0
 840         runner.secret_store = cwltool.secrets.SecretStore()
 841         runner.api._rootDesc = {"revision": "20210628"}
 842
 843         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 844         runner.api.collections().get().execute.return_value = {
 845             "portable_data_hash": "99999999999999999999999999999993+99"}
 846
 847         tool = cmap({
 848             "inputs": [],
 849             "outputs": [],
 850             "baseCommand": "ls",
 851             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 852             "id": "",
 853             "cwlVersion": "v1.2",
 854             "class": "CommandLineTool",
 855             "hints": [
 856                 {
 857                     "class": "ToolTimeLimit",
 858                     "timelimit": 42
 859                 }
 860             ]
 861         })
 862
 863         loadingContext, runtimeContext = self.helper(runner)
 864         runtimeContext.name = "test_timelimit"
 865
 866         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 867         arvtool.formatgraph = None
 868
 869         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 870             j.run(runtimeContext)
 871
 872         _, kwargs = runner.api.container_requests().create.call_args
 873         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 874
 875
 876     # The test passes no builder.resources
 877     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 878     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 879     def test_setting_storage_class(self, keepdocker):
 880         arv_docker_clear_cache()
 881
 882         runner = mock.MagicMock()
 883         runner.ignore_docker_for_reuse = False
 884         runner.intermediate_output_ttl = 0
 885         runner.secret_store = cwltool.secrets.SecretStore()
 886         runner.api._rootDesc = {"revision": "20210628"}
 887
 888         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 889         runner.api.collections().get().execute.return_value = {
 890             "portable_data_hash": "99999999999999999999999999999993+99"}
 891
 892         tool = cmap({
 893             "inputs": [],
 894             "outputs": [],
 895             "baseCommand": "ls",
 896             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 897             "id": "",
 898             "cwlVersion": "v1.2",
 899             "class": "CommandLineTool",
 900             "hints": [
 901                 {
 902                     "class": "http://arvados.org/cwl#OutputStorageClass",
 903                     "finalStorageClass": ["baz_sc", "qux_sc"],
 904                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 905                 }
 906             ]
 907         })
 908
 909         loadingContext, runtimeContext = self.helper(runner, True)
 910
 911         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 912         arvtool.formatgraph = None
 913
 914         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 915             j.run(runtimeContext)
 916             runner.api.container_requests().create.assert_called_with(
 917                 body=JsonDiffMatcher({
 918                     'environment': {
 919                         'HOME': '/var/spool/cwl',
 920                         'TMPDIR': '/tmp'
 921                     },
 922                     'name': 'test_run_True',
 923                     'runtime_constraints': {
 924                         'vcpus': 1,
 925                         'ram': 268435456
 926                     },
 927                     'use_existing': True,
 928                     'priority': 500,
 929                     'mounts': {
 930                         '/tmp': {'kind': 'tmp',
 931                                  "capacity": 1073741824
 932                              },
 933                         '/var/spool/cwl': {'kind': 'tmp',
 934                                            "capacity": 1073741824 }
 935                     },
 936                     'state': 'Committed',
 937                     'output_name': 'Output for step test_run_True',
 938                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 939                     'output_path': '/var/spool/cwl',
 940                     'output_ttl': 0,
 941                     'container_image': '99999999999999999999999999999993+99',
 942                     'command': ['ls', '/var/spool/cwl'],
 943                     'cwd': '/var/spool/cwl',
 944                     'scheduling_parameters': {},
 945                     'properties': {},
 946                     'secret_mounts': {},
 947                     'output_storage_classes': ["foo_sc", "bar_sc"]
 948                 }))
 949
 950
 951     # The test passes no builder.resources
 952     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 953     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 954     def test_setting_process_properties(self, keepdocker):
 955         arv_docker_clear_cache()
 956
 957         runner = mock.MagicMock()
 958         runner.ignore_docker_for_reuse = False
 959         runner.intermediate_output_ttl = 0
 960         runner.secret_store = cwltool.secrets.SecretStore()
 961         runner.api._rootDesc = {"revision": "20210628"}
 962
 963         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 964         runner.api.collections().get().execute.return_value = {
 965             "portable_data_hash": "99999999999999999999999999999993+99"}
 966
 967         tool = cmap({
 968             "inputs": [
 969                 {"id": "x", "type": "string"}],
 970             "outputs": [],
 971             "baseCommand": "ls",
 972             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 973             "id": "",
 974             "class": "CommandLineTool",
 975             "cwlVersion": "v1.2",
 976             "hints": [
 977             {
 978                 "class": "http://arvados.org/cwl#ProcessProperties",
 979                 "processProperties": [
 980                     {"propertyName": "foo",
 981                      "propertyValue": "bar"},
 982                     {"propertyName": "baz",
 983                      "propertyValue": "$(inputs.x)"},
 984                     {"propertyName": "quux",
 985                      "propertyValue": {
 986                          "q1": 1,
 987                          "q2": 2
 988                      }
 989                     }
 990                 ],
 991             }
 992         ]
 993         })
 994
 995         loadingContext, runtimeContext = self.helper(runner, True)
 996
 997         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 998         arvtool.formatgraph = None
 999
1000         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1001             j.run(runtimeContext)
1002             runner.api.container_requests().create.assert_called_with(
1003                 body=JsonDiffMatcher({
1004                     'environment': {
1005                         'HOME': '/var/spool/cwl',
1006                         'TMPDIR': '/tmp'
1007                     },
1008                     'name': 'test_run_True',
1009                     'runtime_constraints': {
1010                         'vcpus': 1,
1011                         'ram': 268435456
1012                     },
1013                     'use_existing': True,
1014                     'priority': 500,
1015                     'mounts': {
1016                         '/tmp': {'kind': 'tmp',
1017                                  "capacity": 1073741824
1018                              },
1019                         '/var/spool/cwl': {'kind': 'tmp',
1020                                            "capacity": 1073741824 }
1021                     },
1022                     'state': 'Committed',
1023                     'output_name': 'Output for step test_run_True',
1024                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1025                     'output_path': '/var/spool/cwl',
1026                     'output_ttl': 0,
1027                     'container_image': '99999999999999999999999999999993+99',
1028                     'command': ['ls', '/var/spool/cwl'],
1029                     'cwd': '/var/spool/cwl',
1030                     'scheduling_parameters': {},
1031                     'properties': {
1032                         "baz": "blorp",
1033                         "foo": "bar",
1034                         "quux": {
1035                             "q1": 1,
1036                             "q2": 2
1037                         }
1038                     },
1039                     'secret_mounts': {},
1040                     'output_storage_classes': ["default"]
1041                 }))
1042
1043
1044     # The test passes no builder.resources
1045     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1046     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1047     def test_cuda_requirement(self, keepdocker):
1048         arvados_cwl.add_arv_hints()
1049         arv_docker_clear_cache()
1050
1051         runner = mock.MagicMock()
1052         runner.ignore_docker_for_reuse = False
1053         runner.intermediate_output_ttl = 0
1054         runner.secret_store = cwltool.secrets.SecretStore()
1055         runner.api._rootDesc = {"revision": "20210628"}
1056
1057         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1058         runner.api.collections().get().execute.return_value = {
1059             "portable_data_hash": "99999999999999999999999999999993+99"}
1060
1061         test_cwl_req = [{
1062                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1063                 "cudaVersionMin": "11.0",
1064                 "cudaComputeCapability": "9.0",
1065             }, {
1066                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1067                 "cudaVersionMin": "11.0",
1068                 "cudaComputeCapability": "9.0",
1069                 "cudaDeviceCountMin": 2
1070             }, {
1071                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1072                 "cudaVersionMin": "11.0",
1073                 "cudaComputeCapability": ["4.0", "5.0"],
1074                 "cudaDeviceCountMin": 2
1075             }]
1076
1077         test_arv_req = [{
1078             'device_count': 1,
1079             'driver_version': "11.0",
1080             'hardware_capability': "9.0"
1081         }, {
1082             'device_count': 2,
1083             'driver_version': "11.0",
1084             'hardware_capability': "9.0"
1085         }, {
1086             'device_count': 2,
1087             'driver_version': "11.0",
1088             'hardware_capability': "4.0"
1089         }]
1090
1091         for test_case in range(0, len(test_cwl_req)):
1092
1093             tool = cmap({
1094                 "inputs": [],
1095                 "outputs": [],
1096                 "baseCommand": "nvidia-smi",
1097                 "arguments": [],
1098                 "id": "",
1099                 "cwlVersion": "v1.2",
1100                 "class": "CommandLineTool",
1101                 "requirements": [test_cwl_req[test_case]]
1102             })
1103
1104             loadingContext, runtimeContext = self.helper(runner, True)
1105
1106             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1107             arvtool.formatgraph = None
1108
1109             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1110                 j.run(runtimeContext)
1111                 runner.api.container_requests().create.assert_called_with(
1112                     body=JsonDiffMatcher({
1113                         'environment': {
1114                             'HOME': '/var/spool/cwl',
1115                             'TMPDIR': '/tmp'
1116                         },
1117                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1118                         'runtime_constraints': {
1119                             'vcpus': 1,
1120                             'ram': 268435456,
1121                             'cuda': test_arv_req[test_case]
1122                         },
1123                         'use_existing': True,
1124                         'priority': 500,
1125                         'mounts': {
1126                             '/tmp': {'kind': 'tmp',
1127                                      "capacity": 1073741824
1128                                  },
1129                             '/var/spool/cwl': {'kind': 'tmp',
1130                                                "capacity": 1073741824 }
1131                         },
1132                         'state': 'Committed',
1133                         'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1134                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1135                         'output_path': '/var/spool/cwl',
1136                         'output_ttl': 0,
1137                         'container_image': '99999999999999999999999999999993+99',
1138                         'command': ['nvidia-smi'],
1139                         'cwd': '/var/spool/cwl',
1140                         'scheduling_parameters': {},
1141                         'properties': {},
1142                         'secret_mounts': {},
1143                         'output_storage_classes': ["default"]
1144                     }))
1145
1146
1147     # The test passes no builder.resources
1148     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1149     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1150     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1151     def test_match_local_docker(self, keepdocker, determine_image_id):
1152         arvados_cwl.add_arv_hints()
1153         arv_docker_clear_cache()
1154
1155         runner = mock.MagicMock()
1156         runner.ignore_docker_for_reuse = False
1157         runner.intermediate_output_ttl = 0
1158         runner.secret_store = cwltool.secrets.SecretStore()
1159         runner.api._rootDesc = {"revision": "20210628"}
1160
1161         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1162                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1163         determine_image_id.side_effect = lambda x: "123"
1164         def execute(uuid):
1165             ex = mock.MagicMock()
1166             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1167                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1168             ex.execute.return_value = lookup[uuid]
1169             return ex
1170         runner.api.collections().get.side_effect = execute
1171
1172         tool = cmap({
1173             "inputs": [],
1174             "outputs": [],
1175             "baseCommand": "echo",
1176             "arguments": [],
1177             "id": "",
1178             "cwlVersion": "v1.0",
1179             "class": "org.w3id.cwl.cwl.CommandLineTool"
1180         })
1181
1182         loadingContext, runtimeContext = self.helper(runner, True)
1183
1184         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1185         arvtool.formatgraph = None
1186
1187         container_request = {
1188             'environment': {
1189                 'HOME': '/var/spool/cwl',
1190                 'TMPDIR': '/tmp'
1191             },
1192             'name': 'test_run_True',
1193             'runtime_constraints': {
1194                 'vcpus': 1,
1195                 'ram': 1073741824,
1196             },
1197             'use_existing': True,
1198             'priority': 500,
1199             'mounts': {
1200                 '/tmp': {'kind': 'tmp',
1201                          "capacity": 1073741824
1202                          },
1203                 '/var/spool/cwl': {'kind': 'tmp',
1204                                    "capacity": 1073741824 }
1205             },
1206             'state': 'Committed',
1207             'output_name': 'Output for step test_run_True',
1208             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1209             'output_path': '/var/spool/cwl',
1210             'output_ttl': 0,
1211             'container_image': '99999999999999999999999999999994+99',
1212             'command': ['echo'],
1213             'cwd': '/var/spool/cwl',
1214             'scheduling_parameters': {},
1215             'properties': {},
1216             'secret_mounts': {},
1217             'output_storage_classes': ["default"]
1218         }
1219
1220         runtimeContext.match_local_docker = False
1221         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1222             j.run(runtimeContext)
1223             runner.api.container_requests().create.assert_called_with(
1224                 body=JsonDiffMatcher(container_request))
1225
1226         arv_docker_clear_cache()
1227         runtimeContext.match_local_docker = True
1228         container_request['container_image'] = '99999999999999999999999999999993+99'
1229         container_request['name'] = 'test_run_True_2'
1230         container_request['output_name'] = 'Output for step test_run_True_2'
1231         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1232             j.run(runtimeContext)
1233             runner.api.container_requests().create.assert_called_with(
1234                 body=JsonDiffMatcher(container_request))
1235
1236
1237     # The test passes no builder.resources
1238     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1239     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1240     def test_run_preemptible_hint(self, keepdocker):
1241         arvados_cwl.add_arv_hints()
1242         for enable_preemptible in (None, True, False):
1243             for preemptible_hint in (None, True, False):
1244                 arv_docker_clear_cache()
1245
1246                 runner = mock.MagicMock()
1247                 runner.ignore_docker_for_reuse = False
1248                 runner.intermediate_output_ttl = 0
1249                 runner.secret_store = cwltool.secrets.SecretStore()
1250                 runner.api._rootDesc = {"revision": "20210628"}
1251
1252                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1253                 runner.api.collections().get().execute.return_value = {
1254                     "portable_data_hash": "99999999999999999999999999999993+99"}
1255
1256                 if preemptible_hint is not None:
1257                     hints = [{
1258                         "class": "http://arvados.org/cwl#UsePreemptible",
1259                         "usePreemptible": preemptible_hint
1260                     }]
1261                 else:
1262                     hints = []
1263
1264                 tool = cmap({
1265                     "inputs": [],
1266                     "outputs": [],
1267                     "baseCommand": "ls",
1268                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1269                     "id": "",
1270                     "class": "CommandLineTool",
1271                     "cwlVersion": "v1.2",
1272                     "hints": hints
1273                 })
1274
1275                 loadingContext, runtimeContext = self.helper(runner)
1276
1277                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1278                 runtimeContext.enable_preemptible = enable_preemptible
1279
1280                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1281                 arvtool.formatgraph = None
1282
1283                 # Test the interactions between --enable/disable-preemptible
1284                 # and UsePreemptible hint
1285
1286                 if enable_preemptible is None:
1287                     if preemptible_hint is None:
1288                         sched = {}
1289                     else:
1290                         sched = {'preemptible': preemptible_hint}
1291                 else:
1292                     if preemptible_hint is None:
1293                         sched = {'preemptible': enable_preemptible}
1294                     else:
1295                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1296
1297                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1298                     j.run(runtimeContext)
1299                     runner.api.container_requests().create.assert_called_with(
1300                         body=JsonDiffMatcher({
1301                             'environment': {
1302                                 'HOME': '/var/spool/cwl',
1303                                 'TMPDIR': '/tmp'
1304                             },
1305                             'name': runtimeContext.name,
1306                             'runtime_constraints': {
1307                                 'vcpus': 1,
1308                                 'ram': 268435456
1309                             },
1310                             'use_existing': True,
1311                             'priority': 500,
1312                             'mounts': {
1313                                 '/tmp': {'kind': 'tmp',
1314                                          "capacity": 1073741824
1315                                      },
1316                                 '/var/spool/cwl': {'kind': 'tmp',
1317                                                    "capacity": 1073741824 }
1318                             },
1319                             'state': 'Committed',
1320                             'output_name': 'Output for step '+runtimeContext.name,
1321                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1322                             'output_path': '/var/spool/cwl',
1323                             'output_ttl': 0,
1324                             'container_image': '99999999999999999999999999999993+99',
1325                             'command': ['ls', '/var/spool/cwl'],
1326                             'cwd': '/var/spool/cwl',
1327                             'scheduling_parameters': sched,
1328                             'properties': {},
1329                             'secret_mounts': {},
1330                             'output_storage_classes': ["default"]
1331                         }))
1332
1333
1334
1335 class TestWorkflow(unittest.TestCase):
1336     def setUp(self):
1337         cwltool.process._names = set()
1338         arv_docker_clear_cache()
1339
1340     def helper(self, runner, enable_reuse=True):
1341         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1342
1343         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1344                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1345
1346         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1347         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1348         document_loader.fetch_text = document_loader.fetcher.fetch_text
1349         document_loader.check_exists = document_loader.fetcher.check_exists
1350
1351         loadingContext = arvados_cwl.context.ArvLoadingContext(
1352             {"avsc_names": avsc_names,
1353              "basedir": "",
1354              "make_fs_access": make_fs_access,
1355              "loader": document_loader,
1356              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1357              "construct_tool_object": runner.arv_make_tool})
1358         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1359             {"work_api": "containers",
1360              "basedir": "",
1361              "name": "test_run_wf_"+str(enable_reuse),
1362              "make_fs_access": make_fs_access,
1363              "tmpdir": "/tmp",
1364              "enable_reuse": enable_reuse,
1365              "priority": 500})
1366
1367         return loadingContext, runtimeContext
1368
1369     # The test passes no builder.resources
1370     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1371     @mock.patch("arvados.collection.CollectionReader")
1372     @mock.patch("arvados.collection.Collection")
1373     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1374     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1375         arvados_cwl.add_arv_hints()
1376
1377         api = mock.MagicMock()
1378         api._rootDesc = get_rootDesc()
1379
1380         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1381         self.assertEqual(runner.work_api, 'containers')
1382
1383         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1384         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1385         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1386                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1387
1388         runner.api.containers().current().execute.return_value = {}
1389
1390         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1391         runner.ignore_docker_for_reuse = False
1392         runner.num_retries = 0
1393         runner.secret_store = cwltool.secrets.SecretStore()
1394
1395         loadingContext, runtimeContext = self.helper(runner)
1396         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1397
1398         mockcollectionreader().exists.return_value = True
1399
1400         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1401         metadata["cwlVersion"] = tool["cwlVersion"]
1402
1403         mockc = mock.MagicMock()
1404         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1405         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1406
1407         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1408         arvtool.formatgraph = None
1409         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1410
1411         next(it).run(runtimeContext)
1412         next(it).run(runtimeContext)
1413
1414         with open("tests/wf/scatter2_subwf.cwl") as f:
1415             subwf = StripYAMLComments(f.read()).rstrip()
1416
1417         runner.api.container_requests().create.assert_called_with(
1418             body=JsonDiffMatcher({
1419                 "command": [
1420                     "cwltool",
1421                     "--no-container",
1422                     "--move-outputs",
1423                     "--preserve-entire-environment",
1424                     "workflow.cwl",
1425                     "cwl.input.yml"
1426                 ],
1427                 "container_image": "99999999999999999999999999999993+99",
1428                 "cwd": "/var/spool/cwl",
1429                 "environment": {
1430                     "HOME": "/var/spool/cwl",
1431                     "TMPDIR": "/tmp"
1432                 },
1433                 "mounts": {
1434                     "/keep/99999999999999999999999999999999+118": {
1435                         "kind": "collection",
1436                         "portable_data_hash": "99999999999999999999999999999999+118"
1437                     },
1438                     "/tmp": {
1439                         "capacity": 1073741824,
1440                         "kind": "tmp"
1441                     },
1442                     "/var/spool/cwl": {
1443                         "capacity": 1073741824,
1444                         "kind": "tmp"
1445                     },
1446                     "/var/spool/cwl/cwl.input.yml": {
1447                         "kind": "collection",
1448                         "path": "cwl.input.yml",
1449                         "portable_data_hash": "99999999999999999999999999999996+99"
1450                     },
1451                     "/var/spool/cwl/workflow.cwl": {
1452                         "kind": "collection",
1453                         "path": "workflow.cwl",
1454                         "portable_data_hash": "99999999999999999999999999999996+99"
1455                     },
1456                     "stdout": {
1457                         "kind": "file",
1458                         "path": "/var/spool/cwl/cwl.output.json"
1459                     }
1460                 },
1461                 "name": "scatterstep",
1462                 "output_name": "Output for step scatterstep",
1463                 "output_path": "/var/spool/cwl",
1464                 "output_ttl": 0,
1465                 "priority": 500,
1466                 "properties": {},
1467                 "runtime_constraints": {
1468                     "ram": 1073741824,
1469                     "vcpus": 1
1470                 },
1471                 "scheduling_parameters": {},
1472                 "secret_mounts": {},
1473                 "state": "Committed",
1474                 "use_existing": True,
1475                 'output_storage_classes': ["default"]
1476             }))
1477         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1478         mockc.open().__enter__().write.assert_has_calls([mock.call(
1479 '''{
1480   "fileblub": {
1481     "basename": "token.txt",
1482     "class": "File",
1483     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1484     "size": 0
1485   },
1486   "sleeptime": 5
1487 }''')])
1488
1489     # The test passes no builder.resources
1490     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1491     @mock.patch("arvados.collection.CollectionReader")
1492     @mock.patch("arvados.collection.Collection")
1493     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1494     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1495         arvados_cwl.add_arv_hints()
1496
1497         api = mock.MagicMock()
1498         api._rootDesc = get_rootDesc()
1499
1500         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1501         self.assertEqual(runner.work_api, 'containers')
1502
1503         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1504         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1505                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1506         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1507                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1508
1509         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1510         runner.ignore_docker_for_reuse = False
1511         runner.num_retries = 0
1512         runner.secret_store = cwltool.secrets.SecretStore()
1513
1514         loadingContext, runtimeContext = self.helper(runner)
1515         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1516         loadingContext.do_update = True
1517         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1518
1519         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1520
1521         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1522         arvtool.formatgraph = None
1523         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1524
1525         next(it).run(runtimeContext)
1526         next(it).run(runtimeContext)
1527
1528         with open("tests/wf/echo-subwf.cwl") as f:
1529             subwf = StripYAMLComments(f.read())
1530
1531         runner.api.container_requests().create.assert_called_with(
1532             body=JsonDiffMatcher({
1533                 'output_ttl': 0,
1534                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1535                 'scheduling_parameters': {},
1536                 'name': u'echo-subwf',
1537                 'secret_mounts': {},
1538                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1539                 'properties': {},
1540                 'priority': 500,
1541                 'mounts': {
1542                     '/var/spool/cwl/cwl.input.yml': {
1543                         'portable_data_hash': '99999999999999999999999999999996+99',
1544                         'kind': 'collection',
1545                         'path': 'cwl.input.yml'
1546                     },
1547                     '/var/spool/cwl/workflow.cwl': {
1548                         'portable_data_hash': '99999999999999999999999999999996+99',
1549                         'kind': 'collection',
1550                         'path': 'workflow.cwl'
1551                     },
1552                     'stdout': {
1553                         'path': '/var/spool/cwl/cwl.output.json',
1554                         'kind': 'file'
1555                     },
1556                     '/tmp': {
1557                         'kind': 'tmp',
1558                         'capacity': 1073741824
1559                     }, '/var/spool/cwl': {
1560                         'kind': 'tmp',
1561                         'capacity': 3221225472
1562                     }
1563                 },
1564                 'state': 'Committed',
1565                 'output_path': '/var/spool/cwl',
1566                 'container_image': '99999999999999999999999999999993+99',
1567                 'command': [
1568                     u'cwltool',
1569                     u'--no-container',
1570                     u'--move-outputs',
1571                     u'--preserve-entire-environment',
1572                     u'workflow.cwl',
1573                     u'cwl.input.yml'
1574                 ],
1575                 'use_existing': True,
1576                 'output_name': u'Output for step echo-subwf',
1577                 'cwd': '/var/spool/cwl',
1578                 'output_storage_classes': ["default"]
1579             }))
1580
1581     def test_default_work_api(self):
1582         arvados_cwl.add_arv_hints()
1583
1584         api = mock.MagicMock()
1585         api._rootDesc = copy.deepcopy(get_rootDesc())
1586         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1587         self.assertEqual(runner.work_api, 'containers')