sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 #from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         #arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  89              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
  90              })
  91         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  92             {"work_api": "containers",
  93              "basedir": "",
  94              "name": "test_run_"+str(enable_reuse),
  95              "make_fs_access": make_fs_access,
  96              "tmpdir": "/tmp",
  97              "outdir": "/tmp",
  98              "enable_reuse": enable_reuse,
  99              "priority": 500,
 100              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 101              "workflow_eval_lock": threading.Condition(threading.RLock())
 102             })
 103
 104         if isinstance(runner, mock.MagicMock):
 105             def make_tool(toolpath_object, loadingContext):
 106                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 107             runner.arv_make_tool.side_effect = make_tool
 108
 109         return loadingContext, runtimeContext
 110
 111     # Helper function to set up the ArvCwlExecutor to use the containers api
 112     # and test that the RuntimeStatusLoggingHandler is set up correctly
 113     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 114         api = mock.MagicMock()
 115         api._rootDesc = copy.deepcopy(get_rootDesc())
 116
 117         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 118         # adds the logging handler that will call runtime_status_update() mock
 119         self.assertFalse(gcc_mock.called)
 120         runner = arvados_cwl.ArvCwlExecutor(api)
 121         self.assertEqual(runner.work_api, 'containers')
 122         root_logger = logging.getLogger('')
 123         handlerClasses = [h.__class__ for h in root_logger.handlers]
 124         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 125         return runner
 126
 127     # The test passes no builder.resources
 128     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 129     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 130     def test_run(self, keepdocker):
 131         for enable_reuse in (True, False):
 132             #arv_docker_clear_cache()
 133
 134             runner = mock.MagicMock()
 135             runner.ignore_docker_for_reuse = False
 136             runner.intermediate_output_ttl = 0
 137             runner.secret_store = cwltool.secrets.SecretStore()
 138             runner.api._rootDesc = {"revision": "20210628"}
 139             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 140
 141             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 142             runner.api.collections().get().execute.return_value = {
 143                 "portable_data_hash": "99999999999999999999999999999993+99"}
 144
 145             tool = cmap({
 146                 "inputs": [],
 147                 "outputs": [],
 148                 "baseCommand": "ls",
 149                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 150                 "id": "",
 151                 "class": "CommandLineTool",
 152                 "cwlVersion": "v1.2"
 153             })
 154
 155             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 156
 157             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 158             arvtool.formatgraph = None
 159
 160             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 161                 j.run(runtimeContext)
 162                 runner.api.container_requests().create.assert_called_with(
 163                     body=JsonDiffMatcher({
 164                         'environment': {
 165                             'HOME': '/var/spool/cwl',
 166                             'TMPDIR': '/tmp'
 167                         },
 168                         'name': 'test_run_'+str(enable_reuse),
 169                         'runtime_constraints': {
 170                             'vcpus': 1,
 171                             'ram': 268435456
 172                         },
 173                         'use_existing': enable_reuse,
 174                         'priority': 500,
 175                         'mounts': {
 176                             '/tmp': {'kind': 'tmp',
 177                                      "capacity": 1073741824
 178                                  },
 179                             '/var/spool/cwl': {'kind': 'tmp',
 180                                                "capacity": 1073741824 }
 181                         },
 182                         'state': 'Committed',
 183                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 184                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 185                         'output_path': '/var/spool/cwl',
 186                         'output_ttl': 0,
 187                         'container_image': '99999999999999999999999999999993+99',
 188                         'command': ['ls', '/var/spool/cwl'],
 189                         'cwd': '/var/spool/cwl',
 190                         'scheduling_parameters': {},
 191                         'properties': {'cwl_input': {}},
 192                         'secret_mounts': {},
 193                         'output_storage_classes': ["default"]
 194                     }))
 195
 196     # The test passes some fields in builder.resources
 197     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 198     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 199     def test_resource_requirements(self, keepdocker):
 200         arvados_cwl.add_arv_hints()
 201         runner = mock.MagicMock()
 202         runner.ignore_docker_for_reuse = False
 203         runner.intermediate_output_ttl = 3600
 204         runner.secret_store = cwltool.secrets.SecretStore()
 205         runner.api._rootDesc = {"revision": "20210628"}
 206         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 207
 208         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 209         runner.api.collections().get().execute.return_value = {
 210             "portable_data_hash": "99999999999999999999999999999993+99"}
 211
 212         tool = cmap({
 213             "inputs": [],
 214             "outputs": [],
 215             "hints": [{
 216                 "class": "ResourceRequirement",
 217                 "coresMin": 3,
 218                 "ramMin": 3000,
 219                 "tmpdirMin": 4000,
 220                 "outdirMin": 5000
 221             }, {
 222                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 223                 "keep_cache": 512
 224             }, {
 225                 "class": "http://arvados.org/cwl#APIRequirement",
 226             }, {
 227                 "class": "http://arvados.org/cwl#PartitionRequirement",
 228                 "partition": "blurb"
 229             }, {
 230                 "class": "http://arvados.org/cwl#IntermediateOutput",
 231                 "outputTTL": 7200
 232             }, {
 233                 "class": "WorkReuse",
 234                 "enableReuse": False
 235             }],
 236             "baseCommand": "ls",
 237             "id": "",
 238             "class": "CommandLineTool",
 239             "cwlVersion": "v1.2"
 240         })
 241
 242         loadingContext, runtimeContext = self.helper(runner)
 243         runtimeContext.name = "test_resource_requirements"
 244
 245         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 246         arvtool.formatgraph = None
 247         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 248             j.run(runtimeContext)
 249
 250         call_args, call_kwargs = runner.api.container_requests().create.call_args
 251
 252         call_body_expected = {
 253             'environment': {
 254                 'HOME': '/var/spool/cwl',
 255                 'TMPDIR': '/tmp'
 256             },
 257             'name': 'test_resource_requirements',
 258             'runtime_constraints': {
 259                 'vcpus': 3,
 260                 'ram': 3145728000,
 261                 'keep_cache_ram': 536870912,
 262                 'API': True
 263             },
 264             'use_existing': False,
 265             'priority': 500,
 266             'mounts': {
 267                 '/tmp': {'kind': 'tmp',
 268                          "capacity": 4194304000 },
 269                 '/var/spool/cwl': {'kind': 'tmp',
 270                                    "capacity": 5242880000 }
 271             },
 272             'state': 'Committed',
 273             'output_name': 'Output from step test_resource_requirements',
 274             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 275             'output_path': '/var/spool/cwl',
 276             'output_ttl': 7200,
 277             'container_image': '99999999999999999999999999999993+99',
 278             'command': ['ls'],
 279             'cwd': '/var/spool/cwl',
 280             'scheduling_parameters': {
 281                 'partitions': ['blurb']
 282             },
 283             'properties': {'cwl_input': {}},
 284             'secret_mounts': {},
 285             'output_storage_classes': ["default"]
 286         }
 287
 288         call_body = call_kwargs.get('body', None)
 289         self.assertNotEqual(None, call_body)
 290         for key in call_body:
 291             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 292
 293
 294     # The test passes some fields in builder.resources
 295     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 296     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 297     @mock.patch("arvados.collection.Collection")
 298     def test_initial_work_dir(self, collection_mock, keepdocker):
 299         runner = mock.MagicMock()
 300         runner.ignore_docker_for_reuse = False
 301         runner.intermediate_output_ttl = 0
 302         runner.secret_store = cwltool.secrets.SecretStore()
 303         runner.api._rootDesc = {"revision": "20210628"}
 304         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 305
 306         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 307         runner.api.collections().get().execute.return_value = {
 308             "portable_data_hash": "99999999999999999999999999999993+99"}
 309
 310         sourcemock = mock.MagicMock()
 311         def get_collection_mock(p):
 312             if "/" in p:
 313                 return (sourcemock, p.split("/", 1)[1])
 314             else:
 315                 return (sourcemock, "")
 316         runner.fs_access.get_collection.side_effect = get_collection_mock
 317
 318         vwdmock = mock.MagicMock()
 319         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 320
 321         tool = cmap({
 322             "inputs": [],
 323             "outputs": [],
 324             "hints": [{
 325                 "class": "InitialWorkDirRequirement",
 326                 "listing": [{
 327                     "class": "File",
 328                     "basename": "foo",
 329                     "location": "keep:99999999999999999999999999999995+99/bar"
 330                 },
 331                 {
 332                     "class": "Directory",
 333                     "basename": "foo2",
 334                     "location": "keep:99999999999999999999999999999995+99"
 335                 },
 336                 {
 337                     "class": "File",
 338                     "basename": "filename",
 339                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 340                 },
 341                 {
 342                     "class": "Directory",
 343                     "basename": "subdir",
 344                     "location": "keep:99999999999999999999999999999995+99/subdir"
 345                 }                        ]
 346             }],
 347             "baseCommand": "ls",
 348             "class": "CommandLineTool",
 349             "cwlVersion": "v1.2",
 350             "id": ""
 351         })
 352
 353         loadingContext, runtimeContext = self.helper(runner)
 354         runtimeContext.name = "test_initial_work_dir"
 355
 356         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 357
 358         arvtool.formatgraph = None
 359         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 360             j.run(runtimeContext)
 361
 362         call_args, call_kwargs = runner.api.container_requests().create.call_args
 363
 364         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 367         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 368
 369         call_body_expected = {
 370             'environment': {
 371                 'HOME': '/var/spool/cwl',
 372                 'TMPDIR': '/tmp'
 373             },
 374             'name': 'test_initial_work_dir',
 375             'runtime_constraints': {
 376                 'vcpus': 1,
 377                 'ram': 268435456
 378             },
 379             'use_existing': True,
 380             'priority': 500,
 381             'mounts': {
 382                 '/tmp': {'kind': 'tmp',
 383                          "capacity": 1073741824 },
 384                 '/var/spool/cwl': {'kind': 'tmp',
 385                                    "capacity": 1073741824 },
 386                 '/var/spool/cwl/foo': {
 387                     'kind': 'collection',
 388                     'path': 'foo',
 389                     'portable_data_hash': '99999999999999999999999999999996+99'
 390                 },
 391                 '/var/spool/cwl/foo2': {
 392                     'kind': 'collection',
 393                     'path': 'foo2',
 394                     'portable_data_hash': '99999999999999999999999999999996+99'
 395                 },
 396                 '/var/spool/cwl/filename': {
 397                     'kind': 'collection',
 398                     'path': 'filename',
 399                     'portable_data_hash': '99999999999999999999999999999996+99'
 400                 },
 401                 '/var/spool/cwl/subdir': {
 402                     'kind': 'collection',
 403                     'path': 'subdir',
 404                     'portable_data_hash': '99999999999999999999999999999996+99'
 405                 }
 406             },
 407             'state': 'Committed',
 408             'output_name': 'Output from step test_initial_work_dir',
 409             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 410             'output_path': '/var/spool/cwl',
 411             'output_ttl': 0,
 412             'container_image': '99999999999999999999999999999993+99',
 413             'command': ['ls'],
 414             'cwd': '/var/spool/cwl',
 415             'scheduling_parameters': {
 416             },
 417             'properties': {'cwl_input': {}},
 418             'secret_mounts': {},
 419             'output_storage_classes': ["default"]
 420         }
 421
 422         call_body = call_kwargs.get('body', None)
 423         self.assertNotEqual(None, call_body)
 424         for key in call_body:
 425             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 426
 427
 428     # Test redirecting stdin/stdout/stderr
 429     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 430     def test_redirects(self, keepdocker):
 431         runner = mock.MagicMock()
 432         runner.ignore_docker_for_reuse = False
 433         runner.intermediate_output_ttl = 0
 434         runner.secret_store = cwltool.secrets.SecretStore()
 435         runner.api._rootDesc = {"revision": "20210628"}
 436         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 437
 438         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 439         runner.api.collections().get().execute.return_value = {
 440             "portable_data_hash": "99999999999999999999999999999993+99"}
 441
 442         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 443
 444         tool = cmap({
 445             "inputs": [],
 446             "outputs": [],
 447             "baseCommand": "ls",
 448             "stdout": "stdout.txt",
 449             "stderr": "stderr.txt",
 450             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 451             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 452             "id": "",
 453             "class": "CommandLineTool",
 454             "cwlVersion": "v1.2"
 455         })
 456
 457         loadingContext, runtimeContext = self.helper(runner)
 458         runtimeContext.name = "test_run_redirect"
 459
 460         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 461         arvtool.formatgraph = None
 462         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 463             j.run(runtimeContext)
 464             runner.api.container_requests().create.assert_called_with(
 465                 body=JsonDiffMatcher({
 466                     'environment': {
 467                         'HOME': '/var/spool/cwl',
 468                         'TMPDIR': '/tmp'
 469                     },
 470                     'name': 'test_run_redirect',
 471                     'runtime_constraints': {
 472                         'vcpus': 1,
 473                         'ram': 268435456
 474                     },
 475                     'use_existing': True,
 476                     'priority': 500,
 477                     'mounts': {
 478                         '/tmp': {'kind': 'tmp',
 479                                  "capacity": 1073741824 },
 480                         '/var/spool/cwl': {'kind': 'tmp',
 481                                            "capacity": 1073741824 },
 482                         "stderr": {
 483                             "kind": "file",
 484                             "path": "/var/spool/cwl/stderr.txt"
 485                         },
 486                         "stdin": {
 487                             "kind": "collection",
 488                             "path": "file.txt",
 489                             "portable_data_hash": "99999999999999999999999999999996+99"
 490                         },
 491                         "stdout": {
 492                             "kind": "file",
 493                             "path": "/var/spool/cwl/stdout.txt"
 494                         },
 495                     },
 496                     'state': 'Committed',
 497                     "output_name": "Output from step test_run_redirect",
 498                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 499                     'output_path': '/var/spool/cwl',
 500                     'output_ttl': 0,
 501                     'container_image': '99999999999999999999999999999993+99',
 502                     'command': ['ls', '/var/spool/cwl'],
 503                     'cwd': '/var/spool/cwl',
 504                     'scheduling_parameters': {},
 505                     'properties': {'cwl_input': {}},
 506                     'secret_mounts': {},
 507                     'output_storage_classes': ["default"]
 508                 }))
 509
 510     @mock.patch("arvados.collection.Collection")
 511     def test_done(self, col):
 512         api = mock.MagicMock()
 513
 514         runner = mock.MagicMock()
 515         runner.api = api
 516         runner.num_retries = 0
 517         runner.ignore_docker_for_reuse = False
 518         runner.intermediate_output_ttl = 0
 519         runner.secret_store = cwltool.secrets.SecretStore()
 520
 521         runner.api.containers().get().execute.return_value = {"state":"Complete",
 522                                                               "output": "abc+123",
 523                                                               "exit_code": 0}
 524
 525         col().open.return_value = []
 526
 527         loadingContext, runtimeContext = self.helper(runner)
 528
 529         arvjob = arvados_cwl.ArvadosContainer(runner,
 530                                               runtimeContext,
 531                                               mock.MagicMock(),
 532                                               {},
 533                                               None,
 534                                               [],
 535                                               [],
 536                                               "testjob")
 537         arvjob.output_callback = mock.MagicMock()
 538         arvjob.collect_outputs = mock.MagicMock()
 539         arvjob.successCodes = [0]
 540         arvjob.outdir = "/var/spool/cwl"
 541         arvjob.output_ttl = 3600
 542         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 543
 544         arvjob.collect_outputs.return_value = {"out": "stuff"}
 545
 546         arvjob.done({
 547             "state": "Final",
 548             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 549             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 550             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 551             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 552             "modified_at": "2017-05-26T12:01:22Z",
 553             "properties": {}
 554         })
 555
 556         self.assertFalse(api.collections().create.called)
 557         self.assertFalse(runner.runtime_status_error.called)
 558
 559         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 560         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 561         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 562
 563         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 564                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 565
 566
 567     # Test to make sure we dont call runtime_status_update if we already did
 568     # some where higher up in the call stack
 569     @mock.patch("arvados_cwl.util.get_current_container")
 570     def test_recursive_runtime_status_update(self, gcc_mock):
 571         self.setup_and_test_container_executor_and_logging(gcc_mock)
 572         root_logger = logging.getLogger('')
 573
 574         # get_current_container is invoked when we call runtime_status_update
 575         # so try and log again!
 576         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 577         try:
 578             root_logger.error("First Error")
 579         except RuntimeError:
 580             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 581
 582
 583     # Test to make sure that an exception raised from
 584     # get_current_container doesn't cause the logger to raise an
 585     # exception
 586     @mock.patch("arvados_cwl.util.get_current_container")
 587     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 588         self.setup_and_test_container_executor_and_logging(gcc_mock)
 589         root_logger = logging.getLogger('')
 590
 591         # get_current_container is invoked when we call
 592         # runtime_status_update, it is going to also raise an
 593         # exception.
 594         gcc_mock.side_effect = Exception("Second Error")
 595         try:
 596             root_logger.error("First Error")
 597         except Exception:
 598             self.fail("Exception in logger should not propagate")
 599         self.assertTrue(gcc_mock.called)
 600
 601     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 602     @mock.patch("arvados_cwl.util.get_current_container")
 603     @mock.patch("arvados.collection.CollectionReader")
 604     @mock.patch("arvados.collection.Collection")
 605     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 606         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 607
 608         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 609         self.assertTrue(gcc_mock.called)
 610
 611         runner.num_retries = 0
 612         runner.ignore_docker_for_reuse = False
 613         runner.intermediate_output_ttl = 0
 614         runner.secret_store = cwltool.secrets.SecretStore()
 615         runner.label = mock.MagicMock()
 616         runner.label.return_value = '[container testjob]'
 617
 618         runner.api.containers().get().execute.return_value = {
 619             "state":"Complete",
 620             "output": "abc+123",
 621             "exit_code": 1,
 622             "log": "def+234"
 623         }
 624
 625         col().open.return_value = []
 626
 627         loadingContext, runtimeContext = self.helper(runner)
 628
 629         arvjob = arvados_cwl.ArvadosContainer(runner,
 630                                               runtimeContext,
 631                                               mock.MagicMock(),
 632                                               {},
 633                                               None,
 634                                               [],
 635                                               [],
 636                                               "testjob")
 637         arvjob.output_callback = mock.MagicMock()
 638         arvjob.collect_outputs = mock.MagicMock()
 639         arvjob.successCodes = [0]
 640         arvjob.outdir = "/var/spool/cwl"
 641         arvjob.output_ttl = 3600
 642         arvjob.collect_outputs.return_value = {"out": "stuff"}
 643
 644         arvjob.done({
 645             "state": "Final",
 646             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 647             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 648             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 649             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 650             "modified_at": "2017-05-26T12:01:22Z",
 651             "properties": {}
 652         })
 653
 654         rts_mock.assert_called_with(
 655             'error',
 656             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 657             '  ** log is empty **'
 658         )
 659         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 660
 661     # The test passes no builder.resources
 662     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 663     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 664     def test_mounts(self, keepdocker):
 665         runner = mock.MagicMock()
 666         runner.ignore_docker_for_reuse = False
 667         runner.intermediate_output_ttl = 0
 668         runner.secret_store = cwltool.secrets.SecretStore()
 669         runner.api._rootDesc = {"revision": "20210628"}
 670         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 671
 672         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 673         runner.api.collections().get().execute.return_value = {
 674             "portable_data_hash": "99999999999999999999999999999994+99",
 675             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 676
 677         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 678
 679         tool = cmap({
 680             "inputs": [
 681                 {"id": "p1",
 682                  "type": "Directory"}
 683             ],
 684             "outputs": [],
 685             "baseCommand": "ls",
 686             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 687             "id": "",
 688             "class": "CommandLineTool",
 689             "cwlVersion": "v1.2"
 690         })
 691
 692         loadingContext, runtimeContext = self.helper(runner)
 693         runtimeContext.name = "test_run_mounts"
 694
 695         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 696         arvtool.formatgraph = None
 697         job_order = {
 698             "p1": {
 699                 "class": "Directory",
 700                 "location": "keep:99999999999999999999999999999994+44",
 701                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 702                 "listing": [
 703                     {
 704                         "class": "File",
 705                         "location": "keep:99999999999999999999999999999994+44/file1",
 706                     },
 707                     {
 708                         "class": "File",
 709                         "location": "keep:99999999999999999999999999999994+44/file2",
 710                     }
 711                 ]
 712             }
 713         }
 714         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 715             j.run(runtimeContext)
 716             runner.api.container_requests().create.assert_called_with(
 717                 body=JsonDiffMatcher({
 718                     'environment': {
 719                         'HOME': '/var/spool/cwl',
 720                         'TMPDIR': '/tmp'
 721                     },
 722                     'name': 'test_run_mounts',
 723                     'runtime_constraints': {
 724                         'vcpus': 1,
 725                         'ram': 268435456
 726                     },
 727                     'use_existing': True,
 728                     'priority': 500,
 729                     'mounts': {
 730                         "/keep/99999999999999999999999999999994+44": {
 731                             "kind": "collection",
 732                             "portable_data_hash": "99999999999999999999999999999994+44",
 733                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 734                         },
 735                         '/tmp': {'kind': 'tmp',
 736                                  "capacity": 1073741824 },
 737                         '/var/spool/cwl': {'kind': 'tmp',
 738                                            "capacity": 1073741824 }
 739                     },
 740                     'state': 'Committed',
 741                     'output_name': 'Output from step test_run_mounts',
 742                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 743                     'output_path': '/var/spool/cwl',
 744                     'output_ttl': 0,
 745                     'container_image': '99999999999999999999999999999994+99',
 746                     'command': ['ls', '/var/spool/cwl'],
 747                     'cwd': '/var/spool/cwl',
 748                     'scheduling_parameters': {},
 749                     'properties': {'cwl_input': {
 750                         "p1": {
 751                             "basename": "99999999999999999999999999999994+44",
 752                             "class": "Directory",
 753                             "dirname": "/keep",
 754                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 755                             "listing": [
 756                                 {
 757                                     "basename": "file1",
 758                                     "class": "File",
 759                                     "dirname": "/keep/99999999999999999999999999999994+44",
 760                                     "location": "keep:99999999999999999999999999999994+44/file1",
 761                                     "nameext": "",
 762                                     "nameroot": "file1",
 763                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 764                                     "size": 0
 765                                 },
 766                                 {
 767                                     "basename": "file2",
 768                                     "class": "File",
 769                                     "dirname": "/keep/99999999999999999999999999999994+44",
 770                                     "location": "keep:99999999999999999999999999999994+44/file2",
 771                                     "nameext": "",
 772                                     "nameroot": "file2",
 773                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 774                                     "size": 0
 775                                 }
 776                             ],
 777                             "location": "keep:99999999999999999999999999999994+44",
 778                             "path": "/keep/99999999999999999999999999999994+44"
 779                         }
 780                     }},
 781                     'secret_mounts': {},
 782                     'output_storage_classes': ["default"]
 783                 }))
 784
 785     # The test passes no builder.resources
 786     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 787     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 788     def test_secrets(self, keepdocker):
 789         arvados_cwl.add_arv_hints()
 790         runner = mock.MagicMock()
 791         runner.ignore_docker_for_reuse = False
 792         runner.intermediate_output_ttl = 0
 793         runner.secret_store = cwltool.secrets.SecretStore()
 794         runner.api._rootDesc = {"revision": "20210628"}
 795         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 796
 797         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 798         runner.api.collections().get().execute.return_value = {
 799             "portable_data_hash": "99999999999999999999999999999993+99"}
 800
 801         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 802
 803         tool = cmap({"arguments": ["md5sum", "example.conf"],
 804                      "class": "CommandLineTool",
 805                      "cwlVersion": "v1.2",
 806                      "hints": [
 807                          {
 808                              "class": "http://commonwl.org/cwltool#Secrets",
 809                              "secrets": [
 810                                  "#secret_job.cwl/pw"
 811                              ]
 812                          }
 813                      ],
 814                      "id": "",
 815                      "inputs": [
 816                          {
 817                              "id": "#secret_job.cwl/pw",
 818                              "type": "string"
 819                          }
 820                      ],
 821                      "outputs": [
 822                      ],
 823                      "requirements": [
 824                          {
 825                              "class": "InitialWorkDirRequirement",
 826                              "listing": [
 827                                  {
 828                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 829                                      "entryname": "example.conf"
 830                                  }
 831                              ]
 832                          }
 833                      ]})
 834
 835         loadingContext, runtimeContext = self.helper(runner)
 836         runtimeContext.name = "test_secrets"
 837
 838         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 839         arvtool.formatgraph = None
 840
 841         job_order = {"pw": "blorp"}
 842         runner.secret_store.store(["pw"], job_order)
 843
 844         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 845             j.run(runtimeContext)
 846             runner.api.container_requests().create.assert_called_with(
 847                 body=JsonDiffMatcher({
 848                     'environment': {
 849                         'HOME': '/var/spool/cwl',
 850                         'TMPDIR': '/tmp'
 851                     },
 852                     'name': 'test_secrets',
 853                     'runtime_constraints': {
 854                         'vcpus': 1,
 855                         'ram': 268435456
 856                     },
 857                     'use_existing': True,
 858                     'priority': 500,
 859                     'mounts': {
 860                         '/tmp': {'kind': 'tmp',
 861                                  "capacity": 1073741824
 862                              },
 863                         '/var/spool/cwl': {'kind': 'tmp',
 864                                            "capacity": 1073741824 }
 865                     },
 866                     'state': 'Committed',
 867                     'output_name': 'Output from step test_secrets',
 868                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 869                     'output_path': '/var/spool/cwl',
 870                     'output_ttl': 0,
 871                     'container_image': '99999999999999999999999999999993+99',
 872                     'command': ['md5sum', 'example.conf'],
 873                     'cwd': '/var/spool/cwl',
 874                     'scheduling_parameters': {},
 875                     'properties': {'cwl_input': job_order},
 876                     "secret_mounts": {
 877                         "/var/spool/cwl/example.conf": {
 878                             "content": "username: user\npassword: blorp\n",
 879                             "kind": "text"
 880                         }
 881                     },
 882                     'output_storage_classes': ["default"]
 883                 }))
 884
 885     # The test passes no builder.resources
 886     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 887     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 888     def test_timelimit(self, keepdocker):
 889         runner = mock.MagicMock()
 890         runner.ignore_docker_for_reuse = False
 891         runner.intermediate_output_ttl = 0
 892         runner.secret_store = cwltool.secrets.SecretStore()
 893         runner.api._rootDesc = {"revision": "20210628"}
 894         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 895
 896         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 897         runner.api.collections().get().execute.return_value = {
 898             "portable_data_hash": "99999999999999999999999999999993+99"}
 899
 900         tool = cmap({
 901             "inputs": [],
 902             "outputs": [],
 903             "baseCommand": "ls",
 904             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 905             "id": "",
 906             "cwlVersion": "v1.2",
 907             "class": "CommandLineTool",
 908             "hints": [
 909                 {
 910                     "class": "ToolTimeLimit",
 911                     "timelimit": 42
 912                 }
 913             ]
 914         })
 915
 916         loadingContext, runtimeContext = self.helper(runner)
 917         runtimeContext.name = "test_timelimit"
 918
 919         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 920         arvtool.formatgraph = None
 921
 922         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 923             j.run(runtimeContext)
 924
 925         _, kwargs = runner.api.container_requests().create.call_args
 926         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 927
 928
 929     # The test passes no builder.resources
 930     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 931     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 932     def test_setting_storage_class(self, keepdocker):
 933         #arv_docker_clear_cache()
 934
 935         runner = mock.MagicMock()
 936         runner.ignore_docker_for_reuse = False
 937         runner.intermediate_output_ttl = 0
 938         runner.secret_store = cwltool.secrets.SecretStore()
 939         runner.api._rootDesc = {"revision": "20210628"}
 940         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 941
 942         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 943         runner.api.collections().get().execute.return_value = {
 944             "portable_data_hash": "99999999999999999999999999999993+99"}
 945
 946         tool = cmap({
 947             "inputs": [],
 948             "outputs": [],
 949             "baseCommand": "ls",
 950             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 951             "id": "",
 952             "cwlVersion": "v1.2",
 953             "class": "CommandLineTool",
 954             "hints": [
 955                 {
 956                     "class": "http://arvados.org/cwl#OutputStorageClass",
 957                     "finalStorageClass": ["baz_sc", "qux_sc"],
 958                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 959                 }
 960             ]
 961         })
 962
 963         loadingContext, runtimeContext = self.helper(runner, True)
 964
 965         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 966         arvtool.formatgraph = None
 967
 968         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 969             j.run(runtimeContext)
 970             runner.api.container_requests().create.assert_called_with(
 971                 body=JsonDiffMatcher({
 972                     'environment': {
 973                         'HOME': '/var/spool/cwl',
 974                         'TMPDIR': '/tmp'
 975                     },
 976                     'name': 'test_run_True',
 977                     'runtime_constraints': {
 978                         'vcpus': 1,
 979                         'ram': 268435456
 980                     },
 981                     'use_existing': True,
 982                     'priority': 500,
 983                     'mounts': {
 984                         '/tmp': {'kind': 'tmp',
 985                                  "capacity": 1073741824
 986                              },
 987                         '/var/spool/cwl': {'kind': 'tmp',
 988                                            "capacity": 1073741824 }
 989                     },
 990                     'state': 'Committed',
 991                     'output_name': 'Output from step test_run_True',
 992                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 993                     'output_path': '/var/spool/cwl',
 994                     'output_ttl': 0,
 995                     'container_image': '99999999999999999999999999999993+99',
 996                     'command': ['ls', '/var/spool/cwl'],
 997                     'cwd': '/var/spool/cwl',
 998                     'scheduling_parameters': {},
 999                     'properties': {'cwl_input': {}},
1000                     'secret_mounts': {},
1001                     'output_storage_classes': ["foo_sc", "bar_sc"]
1002                 }))
1003
1004
1005     # The test passes no builder.resources
1006     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1007     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1008     def test_setting_process_properties(self, keepdocker):
1009         #arv_docker_clear_cache()
1010
1011         runner = mock.MagicMock()
1012         runner.ignore_docker_for_reuse = False
1013         runner.intermediate_output_ttl = 0
1014         runner.secret_store = cwltool.secrets.SecretStore()
1015         runner.api._rootDesc = {"revision": "20210628"}
1016         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1017
1018         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1019         runner.api.collections().get().execute.return_value = {
1020             "portable_data_hash": "99999999999999999999999999999993+99"}
1021
1022         tool = cmap({
1023             "inputs": [
1024                 {"id": "x", "type": "string"}],
1025             "outputs": [],
1026             "baseCommand": "ls",
1027             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1028             "id": "",
1029             "class": "CommandLineTool",
1030             "cwlVersion": "v1.2",
1031             "hints": [
1032             {
1033                 "class": "http://arvados.org/cwl#ProcessProperties",
1034                 "processProperties": [
1035                     {"propertyName": "foo",
1036                      "propertyValue": "bar"},
1037                     {"propertyName": "baz",
1038                      "propertyValue": "$(inputs.x)"},
1039                     {"propertyName": "quux",
1040                      "propertyValue": {
1041                          "q1": 1,
1042                          "q2": 2
1043                      }
1044                     }
1045                 ],
1046             }
1047         ]
1048         })
1049
1050         loadingContext, runtimeContext = self.helper(runner, True)
1051
1052         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1053         arvtool.formatgraph = None
1054
1055         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1056             j.run(runtimeContext)
1057             runner.api.container_requests().create.assert_called_with(
1058                 body=JsonDiffMatcher({
1059                     'environment': {
1060                         'HOME': '/var/spool/cwl',
1061                         'TMPDIR': '/tmp'
1062                     },
1063                     'name': 'test_run_True',
1064                     'runtime_constraints': {
1065                         'vcpus': 1,
1066                         'ram': 268435456
1067                     },
1068                     'use_existing': True,
1069                     'priority': 500,
1070                     'mounts': {
1071                         '/tmp': {'kind': 'tmp',
1072                                  "capacity": 1073741824
1073                              },
1074                         '/var/spool/cwl': {'kind': 'tmp',
1075                                            "capacity": 1073741824 }
1076                     },
1077                     'state': 'Committed',
1078                     'output_name': 'Output from step test_run_True',
1079                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1080                     'output_path': '/var/spool/cwl',
1081                     'output_ttl': 0,
1082                     'container_image': '99999999999999999999999999999993+99',
1083                     'command': ['ls', '/var/spool/cwl'],
1084                     'cwd': '/var/spool/cwl',
1085                     'scheduling_parameters': {},
1086                     'properties': {
1087                         "baz": "blorp",
1088                         "cwl_input": {"x": "blorp"},
1089                         "foo": "bar",
1090                         "quux": {
1091                             "q1": 1,
1092                             "q2": 2
1093                         }
1094                     },
1095                     'secret_mounts': {},
1096                     'output_storage_classes': ["default"]
1097                 }))
1098
1099
1100     # The test passes no builder.resources
1101     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1102     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1103     def test_cuda_requirement(self, keepdocker):
1104         arvados_cwl.add_arv_hints()
1105         #arv_docker_clear_cache()
1106
1107         runner = mock.MagicMock()
1108         runner.ignore_docker_for_reuse = False
1109         runner.intermediate_output_ttl = 0
1110         runner.secret_store = cwltool.secrets.SecretStore()
1111         runner.api._rootDesc = {"revision": "20210628"}
1112         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1113
1114         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1115         runner.api.collections().get().execute.return_value = {
1116             "portable_data_hash": "99999999999999999999999999999993+99"}
1117
1118         test_cwl_req = [{
1119                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1120                 "cudaVersionMin": "11.0",
1121                 "cudaComputeCapability": "9.0",
1122             }, {
1123                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1124                 "cudaVersionMin": "11.0",
1125                 "cudaComputeCapability": "9.0",
1126                 "cudaDeviceCountMin": 2
1127             }, {
1128                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1129                 "cudaVersionMin": "11.0",
1130                 "cudaComputeCapability": ["4.0", "5.0"],
1131                 "cudaDeviceCountMin": 2
1132             }]
1133
1134         test_arv_req = [{
1135             'device_count': 1,
1136             'driver_version': "11.0",
1137             'hardware_capability': "9.0"
1138         }, {
1139             'device_count': 2,
1140             'driver_version': "11.0",
1141             'hardware_capability': "9.0"
1142         }, {
1143             'device_count': 2,
1144             'driver_version': "11.0",
1145             'hardware_capability': "4.0"
1146         }]
1147
1148         for test_case in range(0, len(test_cwl_req)):
1149
1150             tool = cmap({
1151                 "inputs": [],
1152                 "outputs": [],
1153                 "baseCommand": "nvidia-smi",
1154                 "arguments": [],
1155                 "id": "",
1156                 "cwlVersion": "v1.2",
1157                 "class": "CommandLineTool",
1158                 "requirements": [test_cwl_req[test_case]]
1159             })
1160
1161             loadingContext, runtimeContext = self.helper(runner, True)
1162
1163             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1164             arvtool.formatgraph = None
1165
1166             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1167                 j.run(runtimeContext)
1168                 runner.api.container_requests().create.assert_called_with(
1169                     body=JsonDiffMatcher({
1170                         'environment': {
1171                             'HOME': '/var/spool/cwl',
1172                             'TMPDIR': '/tmp'
1173                         },
1174                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1175                         'runtime_constraints': {
1176                             'vcpus': 1,
1177                             'ram': 268435456,
1178                             'cuda': test_arv_req[test_case]
1179                         },
1180                         'use_existing': True,
1181                         'priority': 500,
1182                         'mounts': {
1183                             '/tmp': {'kind': 'tmp',
1184                                      "capacity": 1073741824
1185                                  },
1186                             '/var/spool/cwl': {'kind': 'tmp',
1187                                                "capacity": 1073741824 }
1188                         },
1189                         'state': 'Committed',
1190                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1191                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1192                         'output_path': '/var/spool/cwl',
1193                         'output_ttl': 0,
1194                         'container_image': '99999999999999999999999999999993+99',
1195                         'command': ['nvidia-smi'],
1196                         'cwd': '/var/spool/cwl',
1197                         'scheduling_parameters': {},
1198                         'properties': {'cwl_input': {}},
1199                         'secret_mounts': {},
1200                         'output_storage_classes': ["default"]
1201                     }))
1202
1203
1204     # The test passes no builder.resources
1205     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1206     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1207     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1208     def test_match_local_docker(self, keepdocker, determine_image_id):
1209         arvados_cwl.add_arv_hints()
1210
1211         runner = mock.MagicMock()
1212         runner.ignore_docker_for_reuse = False
1213         runner.intermediate_output_ttl = 0
1214         runner.secret_store = cwltool.secrets.SecretStore()
1215         runner.api._rootDesc = {"revision": "20210628"}
1216         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1217
1218         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1219                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1220         determine_image_id.side_effect = lambda x: "123"
1221         def execute(uuid):
1222             ex = mock.MagicMock()
1223             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1224                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1225             ex.execute.return_value = lookup[uuid]
1226             return ex
1227         runner.api.collections().get.side_effect = execute
1228
1229         tool = cmap({
1230             "inputs": [],
1231             "outputs": [],
1232             "baseCommand": "echo",
1233             "arguments": [],
1234             "id": "",
1235             "cwlVersion": "v1.0",
1236             "class": "org.w3id.cwl.cwl.CommandLineTool"
1237         })
1238
1239         loadingContext, runtimeContext = self.helper(runner, True)
1240
1241         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1242         arvtool.formatgraph = None
1243
1244         container_request = {
1245             'environment': {
1246                 'HOME': '/var/spool/cwl',
1247                 'TMPDIR': '/tmp'
1248             },
1249             'name': 'test_run_True',
1250             'runtime_constraints': {
1251                 'vcpus': 1,
1252                 'ram': 1073741824,
1253             },
1254             'use_existing': True,
1255             'priority': 500,
1256             'mounts': {
1257                 '/tmp': {'kind': 'tmp',
1258                          "capacity": 1073741824
1259                          },
1260                 '/var/spool/cwl': {'kind': 'tmp',
1261                                    "capacity": 1073741824 }
1262             },
1263             'state': 'Committed',
1264             'output_name': 'Output from step test_run_True',
1265             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1266             'output_path': '/var/spool/cwl',
1267             'output_ttl': 0,
1268             'container_image': '99999999999999999999999999999994+99',
1269             'command': ['echo'],
1270             'cwd': '/var/spool/cwl',
1271             'scheduling_parameters': {},
1272             'properties': {'cwl_input': {}},
1273             'secret_mounts': {},
1274             'output_storage_classes': ["default"]
1275         }
1276
1277         runtimeContext.match_local_docker = False
1278         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1279             j.run(runtimeContext)
1280             runner.api.container_requests().create.assert_called_with(
1281                 body=JsonDiffMatcher(container_request))
1282
1283         runtimeContext.cached_docker_lookups.clear()
1284         runtimeContext.match_local_docker = True
1285         container_request['container_image'] = '99999999999999999999999999999993+99'
1286         container_request['name'] = 'test_run_True_2'
1287         container_request['output_name'] = 'Output from step test_run_True_2'
1288         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1289             j.run(runtimeContext)
1290             runner.api.container_requests().create.assert_called_with(
1291                 body=JsonDiffMatcher(container_request))
1292
1293
1294     # The test passes no builder.resources
1295     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1296     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1297     def test_run_preemptible_hint(self, keepdocker):
1298         arvados_cwl.add_arv_hints()
1299         for enable_preemptible in (None, True, False):
1300             for preemptible_hint in (None, True, False):
1301                 #arv_docker_clear_cache()
1302
1303                 runner = mock.MagicMock()
1304                 runner.ignore_docker_for_reuse = False
1305                 runner.intermediate_output_ttl = 0
1306                 runner.secret_store = cwltool.secrets.SecretStore()
1307                 runner.api._rootDesc = {"revision": "20210628"}
1308                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1309
1310                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1311                 runner.api.collections().get().execute.return_value = {
1312                     "portable_data_hash": "99999999999999999999999999999993+99"}
1313
1314                 if preemptible_hint is not None:
1315                     hints = [{
1316                         "class": "http://arvados.org/cwl#UsePreemptible",
1317                         "usePreemptible": preemptible_hint
1318                     }]
1319                 else:
1320                     hints = []
1321
1322                 tool = cmap({
1323                     "inputs": [],
1324                     "outputs": [],
1325                     "baseCommand": "ls",
1326                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1327                     "id": "",
1328                     "class": "CommandLineTool",
1329                     "cwlVersion": "v1.2",
1330                     "hints": hints
1331                 })
1332
1333                 loadingContext, runtimeContext = self.helper(runner)
1334
1335                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1336                 runtimeContext.enable_preemptible = enable_preemptible
1337
1338                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1339                 arvtool.formatgraph = None
1340
1341                 # Test the interactions between --enable/disable-preemptible
1342                 # and UsePreemptible hint
1343
1344                 if enable_preemptible is None:
1345                     if preemptible_hint is None:
1346                         sched = {}
1347                     else:
1348                         sched = {'preemptible': preemptible_hint}
1349                 else:
1350                     if preemptible_hint is None:
1351                         sched = {'preemptible': enable_preemptible}
1352                     else:
1353                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1354
1355                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1356                     j.run(runtimeContext)
1357                     runner.api.container_requests().create.assert_called_with(
1358                         body=JsonDiffMatcher({
1359                             'environment': {
1360                                 'HOME': '/var/spool/cwl',
1361                                 'TMPDIR': '/tmp'
1362                             },
1363                             'name': runtimeContext.name,
1364                             'runtime_constraints': {
1365                                 'vcpus': 1,
1366                                 'ram': 268435456
1367                             },
1368                             'use_existing': True,
1369                             'priority': 500,
1370                             'mounts': {
1371                                 '/tmp': {'kind': 'tmp',
1372                                          "capacity": 1073741824
1373                                      },
1374                                 '/var/spool/cwl': {'kind': 'tmp',
1375                                                    "capacity": 1073741824 }
1376                             },
1377                             'state': 'Committed',
1378                             'output_name': 'Output from step '+runtimeContext.name,
1379                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1380                             'output_path': '/var/spool/cwl',
1381                             'output_ttl': 0,
1382                             'container_image': '99999999999999999999999999999993+99',
1383                             'command': ['ls', '/var/spool/cwl'],
1384                             'cwd': '/var/spool/cwl',
1385                             'scheduling_parameters': sched,
1386                             'properties': {'cwl_input': {}},
1387                             'secret_mounts': {},
1388                             'output_storage_classes': ["default"]
1389                         }))
1390
1391
1392     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1393     def test_output_properties(self, keepdocker):
1394         arvados_cwl.add_arv_hints()
1395         for rev in ["20210628", "20220510"]:
1396             runner = mock.MagicMock()
1397             runner.ignore_docker_for_reuse = False
1398             runner.intermediate_output_ttl = 0
1399             runner.secret_store = cwltool.secrets.SecretStore()
1400             runner.api._rootDesc = {"revision": rev}
1401             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1402
1403             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1404             runner.api.collections().get().execute.return_value = {
1405                 "portable_data_hash": "99999999999999999999999999999993+99"}
1406
1407             tool = cmap({
1408                 "inputs": [{
1409                     "id": "inp",
1410                     "type": "string"
1411                 }],
1412                 "outputs": [],
1413                 "baseCommand": "ls",
1414                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1415                 "id": "",
1416                 "cwlVersion": "v1.2",
1417                 "class": "CommandLineTool",
1418                 "hints": [
1419                     {
1420                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1421                         "outputProperties": {
1422                             "foo": "bar",
1423                             "baz": "$(inputs.inp)"
1424                         }
1425                     }
1426                 ]
1427             })
1428
1429             loadingContext, runtimeContext = self.helper(runner)
1430             runtimeContext.name = "test_timelimit"
1431
1432             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1433             arvtool.formatgraph = None
1434
1435             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1436                 j.run(runtimeContext)
1437
1438             _, kwargs = runner.api.container_requests().create.call_args
1439             if rev == "20220510":
1440                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1441             else:
1442                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1443
1444
1445 class TestWorkflow(unittest.TestCase):
1446     def setUp(self):
1447         cwltool.process._names = set()
1448         #arv_docker_clear_cache()
1449
1450     def helper(self, runner, enable_reuse=True):
1451         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1452
1453         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1454                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1455
1456         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1457         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1458         document_loader.fetch_text = document_loader.fetcher.fetch_text
1459         document_loader.check_exists = document_loader.fetcher.check_exists
1460
1461         loadingContext = arvados_cwl.context.ArvLoadingContext(
1462             {"avsc_names": avsc_names,
1463              "basedir": "",
1464              "make_fs_access": make_fs_access,
1465              "loader": document_loader,
1466              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1467              "construct_tool_object": runner.arv_make_tool,
1468              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
1469         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1470             {"work_api": "containers",
1471              "basedir": "",
1472              "name": "test_run_wf_"+str(enable_reuse),
1473              "make_fs_access": make_fs_access,
1474              "tmpdir": "/tmp",
1475              "enable_reuse": enable_reuse,
1476              "priority": 500})
1477
1478         return loadingContext, runtimeContext
1479
1480     # The test passes no builder.resources
1481     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1482     @mock.patch("arvados.collection.CollectionReader")
1483     @mock.patch("arvados.collection.Collection")
1484     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1485     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1486         arvados_cwl.add_arv_hints()
1487
1488         api = mock.MagicMock()
1489         api._rootDesc = get_rootDesc()
1490         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1491
1492         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1493         self.assertEqual(runner.work_api, 'containers')
1494
1495         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1496         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1497         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1498                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1499
1500         runner.api.containers().current().execute.return_value = {}
1501
1502         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1503         runner.ignore_docker_for_reuse = False
1504         runner.num_retries = 0
1505         runner.secret_store = cwltool.secrets.SecretStore()
1506
1507         loadingContext, runtimeContext = self.helper(runner)
1508         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1509
1510         mockcollectionreader().exists.return_value = True
1511
1512         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1513         metadata["cwlVersion"] = tool["cwlVersion"]
1514
1515         mockc = mock.MagicMock()
1516         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1517         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1518
1519         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1520         arvtool.formatgraph = None
1521         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1522
1523         next(it).run(runtimeContext)
1524         next(it).run(runtimeContext)
1525
1526         with open("tests/wf/scatter2_subwf.cwl") as f:
1527             subwf = StripYAMLComments(f.read()).rstrip()
1528
1529         runner.api.container_requests().create.assert_called_with(
1530             body=JsonDiffMatcher({
1531                 "command": [
1532                     "cwltool",
1533                     "--no-container",
1534                     "--move-outputs",
1535                     "--preserve-entire-environment",
1536                     "workflow.cwl",
1537                     "cwl.input.yml"
1538                 ],
1539                 "container_image": "99999999999999999999999999999993+99",
1540                 "cwd": "/var/spool/cwl",
1541                 "environment": {
1542                     "HOME": "/var/spool/cwl",
1543                     "TMPDIR": "/tmp"
1544                 },
1545                 "mounts": {
1546                     "/keep/99999999999999999999999999999999+118": {
1547                         "kind": "collection",
1548                         "portable_data_hash": "99999999999999999999999999999999+118"
1549                     },
1550                     "/tmp": {
1551                         "capacity": 1073741824,
1552                         "kind": "tmp"
1553                     },
1554                     "/var/spool/cwl": {
1555                         "capacity": 1073741824,
1556                         "kind": "tmp"
1557                     },
1558                     "/var/spool/cwl/cwl.input.yml": {
1559                         "kind": "collection",
1560                         "path": "cwl.input.yml",
1561                         "portable_data_hash": "99999999999999999999999999999996+99"
1562                     },
1563                     "/var/spool/cwl/workflow.cwl": {
1564                         "kind": "collection",
1565                         "path": "workflow.cwl",
1566                         "portable_data_hash": "99999999999999999999999999999996+99"
1567                     },
1568                     "stdout": {
1569                         "kind": "file",
1570                         "path": "/var/spool/cwl/cwl.output.json"
1571                     }
1572                 },
1573                 "name": "scatterstep",
1574                 "output_name": "Output from step scatterstep",
1575                 "output_path": "/var/spool/cwl",
1576                 "output_ttl": 0,
1577                 "priority": 500,
1578                 "properties": {'cwl_input': {
1579                         "fileblub": {
1580                             "basename": "token.txt",
1581                             "class": "File",
1582                             "dirname": "/keep/99999999999999999999999999999999+118",
1583                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1584                             "nameext": ".txt",
1585                             "nameroot": "token",
1586                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1587                             "size": 0
1588                         },
1589                         "sleeptime": 5
1590                 }},
1591                 "runtime_constraints": {
1592                     "ram": 1073741824,
1593                     "vcpus": 1
1594                 },
1595                 "scheduling_parameters": {},
1596                 "secret_mounts": {},
1597                 "state": "Committed",
1598                 "use_existing": True,
1599                 'output_storage_classes': ["default"]
1600             }))
1601         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1602         mockc.open().__enter__().write.assert_has_calls([mock.call(
1603 '''{
1604   "fileblub": {
1605     "basename": "token.txt",
1606     "class": "File",
1607     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1608     "size": 0
1609   },
1610   "sleeptime": 5
1611 }''')])
1612
1613     # The test passes no builder.resources
1614     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1615     @mock.patch("arvados.collection.CollectionReader")
1616     @mock.patch("arvados.collection.Collection")
1617     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1618     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1619         arvados_cwl.add_arv_hints()
1620
1621         api = mock.MagicMock()
1622         api._rootDesc = get_rootDesc()
1623         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1624
1625         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1626         self.assertEqual(runner.work_api, 'containers')
1627
1628         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1629         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1630                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1631         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1632                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1633
1634         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1635         runner.ignore_docker_for_reuse = False
1636         runner.num_retries = 0
1637         runner.secret_store = cwltool.secrets.SecretStore()
1638
1639         loadingContext, runtimeContext = self.helper(runner)
1640         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1641         loadingContext.do_update = True
1642         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1643
1644         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1645
1646         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1647         arvtool.formatgraph = None
1648         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1649
1650         next(it).run(runtimeContext)
1651         next(it).run(runtimeContext)
1652
1653         with open("tests/wf/echo-subwf.cwl") as f:
1654             subwf = StripYAMLComments(f.read())
1655
1656         runner.api.container_requests().create.assert_called_with(
1657             body=JsonDiffMatcher({
1658                 'output_ttl': 0,
1659                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1660                 'scheduling_parameters': {},
1661                 'name': u'echo-subwf',
1662                 'secret_mounts': {},
1663                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1664                 'properties': {'cwl_input': {}},
1665                 'priority': 500,
1666                 'mounts': {
1667                     '/var/spool/cwl/cwl.input.yml': {
1668                         'portable_data_hash': '99999999999999999999999999999996+99',
1669                         'kind': 'collection',
1670                         'path': 'cwl.input.yml'
1671                     },
1672                     '/var/spool/cwl/workflow.cwl': {
1673                         'portable_data_hash': '99999999999999999999999999999996+99',
1674                         'kind': 'collection',
1675                         'path': 'workflow.cwl'
1676                     },
1677                     'stdout': {
1678                         'path': '/var/spool/cwl/cwl.output.json',
1679                         'kind': 'file'
1680                     },
1681                     '/tmp': {
1682                         'kind': 'tmp',
1683                         'capacity': 1073741824
1684                     }, '/var/spool/cwl': {
1685                         'kind': 'tmp',
1686                         'capacity': 3221225472
1687                     }
1688                 },
1689                 'state': 'Committed',
1690                 'output_path': '/var/spool/cwl',
1691                 'container_image': '99999999999999999999999999999993+99',
1692                 'command': [
1693                     u'cwltool',
1694                     u'--no-container',
1695                     u'--move-outputs',
1696                     u'--preserve-entire-environment',
1697                     u'workflow.cwl',
1698                     u'cwl.input.yml'
1699                 ],
1700                 'use_existing': True,
1701                 'output_name': u'Output from step echo-subwf',
1702                 'cwd': '/var/spool/cwl',
1703                 'output_storage_classes': ["default"]
1704             }))
1705
1706     def test_default_work_api(self):
1707         arvados_cwl.add_arv_hints()
1708
1709         api = mock.MagicMock()
1710         api._rootDesc = copy.deepcopy(get_rootDesc())
1711         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1712         self.assertEqual(runner.work_api, 'containers')