sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {'cwl_input': {}},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         tool = cmap({
 210             "inputs": [],
 211             "outputs": [],
 212             "hints": [{
 213                 "class": "ResourceRequirement",
 214                 "coresMin": 3,
 215                 "ramMin": 3000,
 216                 "tmpdirMin": 4000,
 217                 "outdirMin": 5000
 218             }, {
 219                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 220                 "keep_cache": 512
 221             }, {
 222                 "class": "http://arvados.org/cwl#APIRequirement",
 223             }, {
 224                 "class": "http://arvados.org/cwl#PartitionRequirement",
 225                 "partition": "blurb"
 226             }, {
 227                 "class": "http://arvados.org/cwl#IntermediateOutput",
 228                 "outputTTL": 7200
 229             }, {
 230                 "class": "WorkReuse",
 231                 "enableReuse": False
 232             }],
 233             "baseCommand": "ls",
 234             "id": "",
 235             "class": "CommandLineTool",
 236             "cwlVersion": "v1.2"
 237         })
 238
 239         loadingContext, runtimeContext = self.helper(runner)
 240         runtimeContext.name = "test_resource_requirements"
 241
 242         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 243         arvtool.formatgraph = None
 244         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 245             j.run(runtimeContext)
 246
 247         call_args, call_kwargs = runner.api.container_requests().create.call_args
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_resource_requirements',
 255             'runtime_constraints': {
 256                 'vcpus': 3,
 257                 'ram': 3145728000,
 258                 'keep_cache_ram': 536870912,
 259                 'API': True
 260             },
 261             'use_existing': False,
 262             'priority': 500,
 263             'mounts': {
 264                 '/tmp': {'kind': 'tmp',
 265                          "capacity": 4194304000 },
 266                 '/var/spool/cwl': {'kind': 'tmp',
 267                                    "capacity": 5242880000 }
 268             },
 269             'state': 'Committed',
 270             'output_name': 'Output from step test_resource_requirements',
 271             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272             'output_path': '/var/spool/cwl',
 273             'output_ttl': 7200,
 274             'container_image': '99999999999999999999999999999993+99',
 275             'command': ['ls'],
 276             'cwd': '/var/spool/cwl',
 277             'scheduling_parameters': {
 278                 'partitions': ['blurb']
 279             },
 280             'properties': {'cwl_input': {}},
 281             'secret_mounts': {},
 282             'output_storage_classes': ["default"]
 283         }
 284
 285         call_body = call_kwargs.get('body', None)
 286         self.assertNotEqual(None, call_body)
 287         for key in call_body:
 288             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 289
 290
 291     # The test passes some fields in builder.resources
 292     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 293     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 294     @mock.patch("arvados.collection.Collection")
 295     def test_initial_work_dir(self, collection_mock, keepdocker):
 296         runner = mock.MagicMock()
 297         runner.ignore_docker_for_reuse = False
 298         runner.intermediate_output_ttl = 0
 299         runner.secret_store = cwltool.secrets.SecretStore()
 300         runner.api._rootDesc = {"revision": "20210628"}
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         sourcemock = mock.MagicMock()
 307         def get_collection_mock(p):
 308             if "/" in p:
 309                 return (sourcemock, p.split("/", 1)[1])
 310             else:
 311                 return (sourcemock, "")
 312         runner.fs_access.get_collection.side_effect = get_collection_mock
 313
 314         vwdmock = mock.MagicMock()
 315         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 316
 317         tool = cmap({
 318             "inputs": [],
 319             "outputs": [],
 320             "hints": [{
 321                 "class": "InitialWorkDirRequirement",
 322                 "listing": [{
 323                     "class": "File",
 324                     "basename": "foo",
 325                     "location": "keep:99999999999999999999999999999995+99/bar"
 326                 },
 327                 {
 328                     "class": "Directory",
 329                     "basename": "foo2",
 330                     "location": "keep:99999999999999999999999999999995+99"
 331                 },
 332                 {
 333                     "class": "File",
 334                     "basename": "filename",
 335                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 336                 },
 337                 {
 338                     "class": "Directory",
 339                     "basename": "subdir",
 340                     "location": "keep:99999999999999999999999999999995+99/subdir"
 341                 }                        ]
 342             }],
 343             "baseCommand": "ls",
 344             "class": "CommandLineTool",
 345             "cwlVersion": "v1.2",
 346             "id": ""
 347         })
 348
 349         loadingContext, runtimeContext = self.helper(runner)
 350         runtimeContext.name = "test_initial_work_dir"
 351
 352         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 353
 354         arvtool.formatgraph = None
 355         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 356             j.run(runtimeContext)
 357
 358         call_args, call_kwargs = runner.api.container_requests().create.call_args
 359
 360         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 361         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 364
 365         call_body_expected = {
 366             'environment': {
 367                 'HOME': '/var/spool/cwl',
 368                 'TMPDIR': '/tmp'
 369             },
 370             'name': 'test_initial_work_dir',
 371             'runtime_constraints': {
 372                 'vcpus': 1,
 373                 'ram': 268435456
 374             },
 375             'use_existing': True,
 376             'priority': 500,
 377             'mounts': {
 378                 '/tmp': {'kind': 'tmp',
 379                          "capacity": 1073741824 },
 380                 '/var/spool/cwl': {'kind': 'tmp',
 381                                    "capacity": 1073741824 },
 382                 '/var/spool/cwl/foo': {
 383                     'kind': 'collection',
 384                     'path': 'foo',
 385                     'portable_data_hash': '99999999999999999999999999999996+99'
 386                 },
 387                 '/var/spool/cwl/foo2': {
 388                     'kind': 'collection',
 389                     'path': 'foo2',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/filename': {
 393                     'kind': 'collection',
 394                     'path': 'filename',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/subdir': {
 398                     'kind': 'collection',
 399                     'path': 'subdir',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 }
 402             },
 403             'state': 'Committed',
 404             'output_name': 'Output from step test_initial_work_dir',
 405             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 406             'output_path': '/var/spool/cwl',
 407             'output_ttl': 0,
 408             'container_image': '99999999999999999999999999999993+99',
 409             'command': ['ls'],
 410             'cwd': '/var/spool/cwl',
 411             'scheduling_parameters': {
 412             },
 413             'properties': {'cwl_input': {}},
 414             'secret_mounts': {},
 415             'output_storage_classes': ["default"]
 416         }
 417
 418         call_body = call_kwargs.get('body', None)
 419         self.assertNotEqual(None, call_body)
 420         for key in call_body:
 421             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 422
 423
 424     # Test redirecting stdin/stdout/stderr
 425     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 426     def test_redirects(self, keepdocker):
 427         runner = mock.MagicMock()
 428         runner.ignore_docker_for_reuse = False
 429         runner.intermediate_output_ttl = 0
 430         runner.secret_store = cwltool.secrets.SecretStore()
 431         runner.api._rootDesc = {"revision": "20210628"}
 432
 433         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 434         runner.api.collections().get().execute.return_value = {
 435             "portable_data_hash": "99999999999999999999999999999993+99"}
 436
 437         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 438
 439         tool = cmap({
 440             "inputs": [],
 441             "outputs": [],
 442             "baseCommand": "ls",
 443             "stdout": "stdout.txt",
 444             "stderr": "stderr.txt",
 445             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 446             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 447             "id": "",
 448             "class": "CommandLineTool",
 449             "cwlVersion": "v1.2"
 450         })
 451
 452         loadingContext, runtimeContext = self.helper(runner)
 453         runtimeContext.name = "test_run_redirect"
 454
 455         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 456         arvtool.formatgraph = None
 457         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 458             j.run(runtimeContext)
 459             runner.api.container_requests().create.assert_called_with(
 460                 body=JsonDiffMatcher({
 461                     'environment': {
 462                         'HOME': '/var/spool/cwl',
 463                         'TMPDIR': '/tmp'
 464                     },
 465                     'name': 'test_run_redirect',
 466                     'runtime_constraints': {
 467                         'vcpus': 1,
 468                         'ram': 268435456
 469                     },
 470                     'use_existing': True,
 471                     'priority': 500,
 472                     'mounts': {
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 },
 477                         "stderr": {
 478                             "kind": "file",
 479                             "path": "/var/spool/cwl/stderr.txt"
 480                         },
 481                         "stdin": {
 482                             "kind": "collection",
 483                             "path": "file.txt",
 484                             "portable_data_hash": "99999999999999999999999999999996+99"
 485                         },
 486                         "stdout": {
 487                             "kind": "file",
 488                             "path": "/var/spool/cwl/stdout.txt"
 489                         },
 490                     },
 491                     'state': 'Committed',
 492                     "output_name": "Output from step test_run_redirect",
 493                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 494                     'output_path': '/var/spool/cwl',
 495                     'output_ttl': 0,
 496                     'container_image': '99999999999999999999999999999993+99',
 497                     'command': ['ls', '/var/spool/cwl'],
 498                     'cwd': '/var/spool/cwl',
 499                     'scheduling_parameters': {},
 500                     'properties': {'cwl_input': {}},
 501                     'secret_mounts': {},
 502                     'output_storage_classes': ["default"]
 503                 }))
 504
 505     @mock.patch("arvados.collection.Collection")
 506     def test_done(self, col):
 507         api = mock.MagicMock()
 508
 509         runner = mock.MagicMock()
 510         runner.api = api
 511         runner.num_retries = 0
 512         runner.ignore_docker_for_reuse = False
 513         runner.intermediate_output_ttl = 0
 514         runner.secret_store = cwltool.secrets.SecretStore()
 515
 516         runner.api.containers().get().execute.return_value = {"state":"Complete",
 517                                                               "output": "abc+123",
 518                                                               "exit_code": 0}
 519
 520         col().open.return_value = []
 521
 522         loadingContext, runtimeContext = self.helper(runner)
 523
 524         arvjob = arvados_cwl.ArvadosContainer(runner,
 525                                               runtimeContext,
 526                                               mock.MagicMock(),
 527                                               {},
 528                                               None,
 529                                               [],
 530                                               [],
 531                                               "testjob")
 532         arvjob.output_callback = mock.MagicMock()
 533         arvjob.collect_outputs = mock.MagicMock()
 534         arvjob.successCodes = [0]
 535         arvjob.outdir = "/var/spool/cwl"
 536         arvjob.output_ttl = 3600
 537         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 538
 539         arvjob.collect_outputs.return_value = {"out": "stuff"}
 540
 541         arvjob.done({
 542             "state": "Final",
 543             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 544             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 545             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 546             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 547             "modified_at": "2017-05-26T12:01:22Z",
 548             "properties": {}
 549         })
 550
 551         self.assertFalse(api.collections().create.called)
 552         self.assertFalse(runner.runtime_status_error.called)
 553
 554         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 555         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 556         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 557
 558         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 559                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 560
 561
 562     # Test to make sure we dont call runtime_status_update if we already did
 563     # some where higher up in the call stack
 564     @mock.patch("arvados_cwl.util.get_current_container")
 565     def test_recursive_runtime_status_update(self, gcc_mock):
 566         self.setup_and_test_container_executor_and_logging(gcc_mock)
 567         root_logger = logging.getLogger('')
 568
 569         # get_current_container is invoked when we call runtime_status_update
 570         # so try and log again!
 571         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 572         try:
 573             root_logger.error("First Error")
 574         except RuntimeError:
 575             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 576
 577
 578     # Test to make sure that an exception raised from
 579     # get_current_container doesn't cause the logger to raise an
 580     # exception
 581     @mock.patch("arvados_cwl.util.get_current_container")
 582     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 583         self.setup_and_test_container_executor_and_logging(gcc_mock)
 584         root_logger = logging.getLogger('')
 585
 586         # get_current_container is invoked when we call
 587         # runtime_status_update, it is going to also raise an
 588         # exception.
 589         gcc_mock.side_effect = Exception("Second Error")
 590         try:
 591             root_logger.error("First Error")
 592         except Exception:
 593             self.fail("Exception in logger should not propagate")
 594         self.assertTrue(gcc_mock.called)
 595
 596     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 597     @mock.patch("arvados_cwl.util.get_current_container")
 598     @mock.patch("arvados.collection.CollectionReader")
 599     @mock.patch("arvados.collection.Collection")
 600     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 601         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 602
 603         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 604         self.assertTrue(gcc_mock.called)
 605
 606         runner.num_retries = 0
 607         runner.ignore_docker_for_reuse = False
 608         runner.intermediate_output_ttl = 0
 609         runner.secret_store = cwltool.secrets.SecretStore()
 610         runner.label = mock.MagicMock()
 611         runner.label.return_value = '[container testjob]'
 612
 613         runner.api.containers().get().execute.return_value = {
 614             "state":"Complete",
 615             "output": "abc+123",
 616             "exit_code": 1,
 617             "log": "def+234"
 618         }
 619
 620         col().open.return_value = []
 621
 622         loadingContext, runtimeContext = self.helper(runner)
 623
 624         arvjob = arvados_cwl.ArvadosContainer(runner,
 625                                               runtimeContext,
 626                                               mock.MagicMock(),
 627                                               {},
 628                                               None,
 629                                               [],
 630                                               [],
 631                                               "testjob")
 632         arvjob.output_callback = mock.MagicMock()
 633         arvjob.collect_outputs = mock.MagicMock()
 634         arvjob.successCodes = [0]
 635         arvjob.outdir = "/var/spool/cwl"
 636         arvjob.output_ttl = 3600
 637         arvjob.collect_outputs.return_value = {"out": "stuff"}
 638
 639         arvjob.done({
 640             "state": "Final",
 641             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 642             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 643             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 644             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 645             "modified_at": "2017-05-26T12:01:22Z",
 646             "properties": {}
 647         })
 648
 649         rts_mock.assert_called_with(
 650             'error',
 651             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 652             '  ** log is empty **'
 653         )
 654         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 655
 656     # The test passes no builder.resources
 657     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 658     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 659     def test_mounts(self, keepdocker):
 660         runner = mock.MagicMock()
 661         runner.ignore_docker_for_reuse = False
 662         runner.intermediate_output_ttl = 0
 663         runner.secret_store = cwltool.secrets.SecretStore()
 664         runner.api._rootDesc = {"revision": "20210628"}
 665
 666         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 667         runner.api.collections().get().execute.return_value = {
 668             "portable_data_hash": "99999999999999999999999999999994+99",
 669             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 670
 671         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 672
 673         tool = cmap({
 674             "inputs": [
 675                 {"id": "p1",
 676                  "type": "Directory"}
 677             ],
 678             "outputs": [],
 679             "baseCommand": "ls",
 680             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 681             "id": "",
 682             "class": "CommandLineTool",
 683             "cwlVersion": "v1.2"
 684         })
 685
 686         loadingContext, runtimeContext = self.helper(runner)
 687         runtimeContext.name = "test_run_mounts"
 688
 689         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 690         arvtool.formatgraph = None
 691         job_order = {
 692             "p1": {
 693                 "class": "Directory",
 694                 "location": "keep:99999999999999999999999999999994+44",
 695                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 696                 "listing": [
 697                     {
 698                         "class": "File",
 699                         "location": "keep:99999999999999999999999999999994+44/file1",
 700                     },
 701                     {
 702                         "class": "File",
 703                         "location": "keep:99999999999999999999999999999994+44/file2",
 704                     }
 705                 ]
 706             }
 707         }
 708         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 709             j.run(runtimeContext)
 710             runner.api.container_requests().create.assert_called_with(
 711                 body=JsonDiffMatcher({
 712                     'environment': {
 713                         'HOME': '/var/spool/cwl',
 714                         'TMPDIR': '/tmp'
 715                     },
 716                     'name': 'test_run_mounts',
 717                     'runtime_constraints': {
 718                         'vcpus': 1,
 719                         'ram': 268435456
 720                     },
 721                     'use_existing': True,
 722                     'priority': 500,
 723                     'mounts': {
 724                         "/keep/99999999999999999999999999999994+44": {
 725                             "kind": "collection",
 726                             "portable_data_hash": "99999999999999999999999999999994+44",
 727                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 728                         },
 729                         '/tmp': {'kind': 'tmp',
 730                                  "capacity": 1073741824 },
 731                         '/var/spool/cwl': {'kind': 'tmp',
 732                                            "capacity": 1073741824 }
 733                     },
 734                     'state': 'Committed',
 735                     'output_name': 'Output from step test_run_mounts',
 736                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 737                     'output_path': '/var/spool/cwl',
 738                     'output_ttl': 0,
 739                     'container_image': '99999999999999999999999999999994+99',
 740                     'command': ['ls', '/var/spool/cwl'],
 741                     'cwd': '/var/spool/cwl',
 742                     'scheduling_parameters': {},
 743                     'properties': {'cwl_input': {
 744                         "p1": {
 745                             "basename": "99999999999999999999999999999994+44",
 746                             "class": "Directory",
 747                             "dirname": "/keep",
 748                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 749                             "listing": [
 750                                 {
 751                                     "basename": "file1",
 752                                     "class": "File",
 753                                     "dirname": "/keep/99999999999999999999999999999994+44",
 754                                     "location": "keep:99999999999999999999999999999994+44/file1",
 755                                     "nameext": "",
 756                                     "nameroot": "file1",
 757                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 758                                     "size": 0
 759                                 },
 760                                 {
 761                                     "basename": "file2",
 762                                     "class": "File",
 763                                     "dirname": "/keep/99999999999999999999999999999994+44",
 764                                     "location": "keep:99999999999999999999999999999994+44/file2",
 765                                     "nameext": "",
 766                                     "nameroot": "file2",
 767                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 768                                     "size": 0
 769                                 }
 770                             ],
 771                             "location": "keep:99999999999999999999999999999994+44",
 772                             "path": "/keep/99999999999999999999999999999994+44"
 773                         }
 774                     }},
 775                     'secret_mounts': {},
 776                     'output_storage_classes': ["default"]
 777                 }))
 778
 779     # The test passes no builder.resources
 780     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 781     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 782     def test_secrets(self, keepdocker):
 783         arvados_cwl.add_arv_hints()
 784         runner = mock.MagicMock()
 785         runner.ignore_docker_for_reuse = False
 786         runner.intermediate_output_ttl = 0
 787         runner.secret_store = cwltool.secrets.SecretStore()
 788         runner.api._rootDesc = {"revision": "20210628"}
 789
 790         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 791         runner.api.collections().get().execute.return_value = {
 792             "portable_data_hash": "99999999999999999999999999999993+99"}
 793
 794         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 795
 796         tool = cmap({"arguments": ["md5sum", "example.conf"],
 797                      "class": "CommandLineTool",
 798                      "cwlVersion": "v1.2",
 799                      "hints": [
 800                          {
 801                              "class": "http://commonwl.org/cwltool#Secrets",
 802                              "secrets": [
 803                                  "#secret_job.cwl/pw"
 804                              ]
 805                          }
 806                      ],
 807                      "id": "",
 808                      "inputs": [
 809                          {
 810                              "id": "#secret_job.cwl/pw",
 811                              "type": "string"
 812                          }
 813                      ],
 814                      "outputs": [
 815                      ],
 816                      "requirements": [
 817                          {
 818                              "class": "InitialWorkDirRequirement",
 819                              "listing": [
 820                                  {
 821                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 822                                      "entryname": "example.conf"
 823                                  }
 824                              ]
 825                          }
 826                      ]})
 827
 828         loadingContext, runtimeContext = self.helper(runner)
 829         runtimeContext.name = "test_secrets"
 830
 831         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 832         arvtool.formatgraph = None
 833
 834         job_order = {"pw": "blorp"}
 835         runner.secret_store.store(["pw"], job_order)
 836
 837         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 838             j.run(runtimeContext)
 839             runner.api.container_requests().create.assert_called_with(
 840                 body=JsonDiffMatcher({
 841                     'environment': {
 842                         'HOME': '/var/spool/cwl',
 843                         'TMPDIR': '/tmp'
 844                     },
 845                     'name': 'test_secrets',
 846                     'runtime_constraints': {
 847                         'vcpus': 1,
 848                         'ram': 268435456
 849                     },
 850                     'use_existing': True,
 851                     'priority': 500,
 852                     'mounts': {
 853                         '/tmp': {'kind': 'tmp',
 854                                  "capacity": 1073741824
 855                              },
 856                         '/var/spool/cwl': {'kind': 'tmp',
 857                                            "capacity": 1073741824 }
 858                     },
 859                     'state': 'Committed',
 860                     'output_name': 'Output from step test_secrets',
 861                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 862                     'output_path': '/var/spool/cwl',
 863                     'output_ttl': 0,
 864                     'container_image': '99999999999999999999999999999993+99',
 865                     'command': ['md5sum', 'example.conf'],
 866                     'cwd': '/var/spool/cwl',
 867                     'scheduling_parameters': {},
 868                     'properties': {'cwl_input': job_order},
 869                     "secret_mounts": {
 870                         "/var/spool/cwl/example.conf": {
 871                             "content": "username: user\npassword: blorp\n",
 872                             "kind": "text"
 873                         }
 874                     },
 875                     'output_storage_classes': ["default"]
 876                 }))
 877
 878     # The test passes no builder.resources
 879     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 880     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 881     def test_timelimit(self, keepdocker):
 882         runner = mock.MagicMock()
 883         runner.ignore_docker_for_reuse = False
 884         runner.intermediate_output_ttl = 0
 885         runner.secret_store = cwltool.secrets.SecretStore()
 886         runner.api._rootDesc = {"revision": "20210628"}
 887
 888         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 889         runner.api.collections().get().execute.return_value = {
 890             "portable_data_hash": "99999999999999999999999999999993+99"}
 891
 892         tool = cmap({
 893             "inputs": [],
 894             "outputs": [],
 895             "baseCommand": "ls",
 896             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 897             "id": "",
 898             "cwlVersion": "v1.2",
 899             "class": "CommandLineTool",
 900             "hints": [
 901                 {
 902                     "class": "ToolTimeLimit",
 903                     "timelimit": 42
 904                 }
 905             ]
 906         })
 907
 908         loadingContext, runtimeContext = self.helper(runner)
 909         runtimeContext.name = "test_timelimit"
 910
 911         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 912         arvtool.formatgraph = None
 913
 914         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 915             j.run(runtimeContext)
 916
 917         _, kwargs = runner.api.container_requests().create.call_args
 918         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 919
 920
 921     # The test passes no builder.resources
 922     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 923     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 924     def test_setting_storage_class(self, keepdocker):
 925         arv_docker_clear_cache()
 926
 927         runner = mock.MagicMock()
 928         runner.ignore_docker_for_reuse = False
 929         runner.intermediate_output_ttl = 0
 930         runner.secret_store = cwltool.secrets.SecretStore()
 931         runner.api._rootDesc = {"revision": "20210628"}
 932
 933         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 934         runner.api.collections().get().execute.return_value = {
 935             "portable_data_hash": "99999999999999999999999999999993+99"}
 936
 937         tool = cmap({
 938             "inputs": [],
 939             "outputs": [],
 940             "baseCommand": "ls",
 941             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 942             "id": "",
 943             "cwlVersion": "v1.2",
 944             "class": "CommandLineTool",
 945             "hints": [
 946                 {
 947                     "class": "http://arvados.org/cwl#OutputStorageClass",
 948                     "finalStorageClass": ["baz_sc", "qux_sc"],
 949                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 950                 }
 951             ]
 952         })
 953
 954         loadingContext, runtimeContext = self.helper(runner, True)
 955
 956         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 957         arvtool.formatgraph = None
 958
 959         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 960             j.run(runtimeContext)
 961             runner.api.container_requests().create.assert_called_with(
 962                 body=JsonDiffMatcher({
 963                     'environment': {
 964                         'HOME': '/var/spool/cwl',
 965                         'TMPDIR': '/tmp'
 966                     },
 967                     'name': 'test_run_True',
 968                     'runtime_constraints': {
 969                         'vcpus': 1,
 970                         'ram': 268435456
 971                     },
 972                     'use_existing': True,
 973                     'priority': 500,
 974                     'mounts': {
 975                         '/tmp': {'kind': 'tmp',
 976                                  "capacity": 1073741824
 977                              },
 978                         '/var/spool/cwl': {'kind': 'tmp',
 979                                            "capacity": 1073741824 }
 980                     },
 981                     'state': 'Committed',
 982                     'output_name': 'Output from step test_run_True',
 983                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 984                     'output_path': '/var/spool/cwl',
 985                     'output_ttl': 0,
 986                     'container_image': '99999999999999999999999999999993+99',
 987                     'command': ['ls', '/var/spool/cwl'],
 988                     'cwd': '/var/spool/cwl',
 989                     'scheduling_parameters': {},
 990                     'properties': {'cwl_input': {}},
 991                     'secret_mounts': {},
 992                     'output_storage_classes': ["foo_sc", "bar_sc"]
 993                 }))
 994
 995
 996     # The test passes no builder.resources
 997     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 998     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 999     def test_setting_process_properties(self, keepdocker):
1000         arv_docker_clear_cache()
1001
1002         runner = mock.MagicMock()
1003         runner.ignore_docker_for_reuse = False
1004         runner.intermediate_output_ttl = 0
1005         runner.secret_store = cwltool.secrets.SecretStore()
1006         runner.api._rootDesc = {"revision": "20210628"}
1007
1008         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1009         runner.api.collections().get().execute.return_value = {
1010             "portable_data_hash": "99999999999999999999999999999993+99"}
1011
1012         tool = cmap({
1013             "inputs": [
1014                 {"id": "x", "type": "string"}],
1015             "outputs": [],
1016             "baseCommand": "ls",
1017             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1018             "id": "",
1019             "class": "CommandLineTool",
1020             "cwlVersion": "v1.2",
1021             "hints": [
1022             {
1023                 "class": "http://arvados.org/cwl#ProcessProperties",
1024                 "processProperties": [
1025                     {"propertyName": "foo",
1026                      "propertyValue": "bar"},
1027                     {"propertyName": "baz",
1028                      "propertyValue": "$(inputs.x)"},
1029                     {"propertyName": "quux",
1030                      "propertyValue": {
1031                          "q1": 1,
1032                          "q2": 2
1033                      }
1034                     }
1035                 ],
1036             }
1037         ]
1038         })
1039
1040         loadingContext, runtimeContext = self.helper(runner, True)
1041
1042         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1043         arvtool.formatgraph = None
1044
1045         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1046             j.run(runtimeContext)
1047             runner.api.container_requests().create.assert_called_with(
1048                 body=JsonDiffMatcher({
1049                     'environment': {
1050                         'HOME': '/var/spool/cwl',
1051                         'TMPDIR': '/tmp'
1052                     },
1053                     'name': 'test_run_True',
1054                     'runtime_constraints': {
1055                         'vcpus': 1,
1056                         'ram': 268435456
1057                     },
1058                     'use_existing': True,
1059                     'priority': 500,
1060                     'mounts': {
1061                         '/tmp': {'kind': 'tmp',
1062                                  "capacity": 1073741824
1063                              },
1064                         '/var/spool/cwl': {'kind': 'tmp',
1065                                            "capacity": 1073741824 }
1066                     },
1067                     'state': 'Committed',
1068                     'output_name': 'Output from step test_run_True',
1069                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1070                     'output_path': '/var/spool/cwl',
1071                     'output_ttl': 0,
1072                     'container_image': '99999999999999999999999999999993+99',
1073                     'command': ['ls', '/var/spool/cwl'],
1074                     'cwd': '/var/spool/cwl',
1075                     'scheduling_parameters': {},
1076                     'properties': {
1077                         "baz": "blorp",
1078                         "cwl_input": {"x": "blorp"},
1079                         "foo": "bar",
1080                         "quux": {
1081                             "q1": 1,
1082                             "q2": 2
1083                         }
1084                     },
1085                     'secret_mounts': {},
1086                     'output_storage_classes': ["default"]
1087                 }))
1088
1089
1090     # The test passes no builder.resources
1091     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1092     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1093     def test_cuda_requirement(self, keepdocker):
1094         arvados_cwl.add_arv_hints()
1095         arv_docker_clear_cache()
1096
1097         runner = mock.MagicMock()
1098         runner.ignore_docker_for_reuse = False
1099         runner.intermediate_output_ttl = 0
1100         runner.secret_store = cwltool.secrets.SecretStore()
1101         runner.api._rootDesc = {"revision": "20210628"}
1102
1103         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1104         runner.api.collections().get().execute.return_value = {
1105             "portable_data_hash": "99999999999999999999999999999993+99"}
1106
1107         test_cwl_req = [{
1108                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1109                 "cudaVersionMin": "11.0",
1110                 "cudaComputeCapability": "9.0",
1111             }, {
1112                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1113                 "cudaVersionMin": "11.0",
1114                 "cudaComputeCapability": "9.0",
1115                 "cudaDeviceCountMin": 2
1116             }, {
1117                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1118                 "cudaVersionMin": "11.0",
1119                 "cudaComputeCapability": ["4.0", "5.0"],
1120                 "cudaDeviceCountMin": 2
1121             }]
1122
1123         test_arv_req = [{
1124             'device_count': 1,
1125             'driver_version': "11.0",
1126             'hardware_capability': "9.0"
1127         }, {
1128             'device_count': 2,
1129             'driver_version': "11.0",
1130             'hardware_capability': "9.0"
1131         }, {
1132             'device_count': 2,
1133             'driver_version': "11.0",
1134             'hardware_capability': "4.0"
1135         }]
1136
1137         for test_case in range(0, len(test_cwl_req)):
1138
1139             tool = cmap({
1140                 "inputs": [],
1141                 "outputs": [],
1142                 "baseCommand": "nvidia-smi",
1143                 "arguments": [],
1144                 "id": "",
1145                 "cwlVersion": "v1.2",
1146                 "class": "CommandLineTool",
1147                 "requirements": [test_cwl_req[test_case]]
1148             })
1149
1150             loadingContext, runtimeContext = self.helper(runner, True)
1151
1152             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1153             arvtool.formatgraph = None
1154
1155             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1156                 j.run(runtimeContext)
1157                 runner.api.container_requests().create.assert_called_with(
1158                     body=JsonDiffMatcher({
1159                         'environment': {
1160                             'HOME': '/var/spool/cwl',
1161                             'TMPDIR': '/tmp'
1162                         },
1163                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1164                         'runtime_constraints': {
1165                             'vcpus': 1,
1166                             'ram': 268435456,
1167                             'cuda': test_arv_req[test_case]
1168                         },
1169                         'use_existing': True,
1170                         'priority': 500,
1171                         'mounts': {
1172                             '/tmp': {'kind': 'tmp',
1173                                      "capacity": 1073741824
1174                                  },
1175                             '/var/spool/cwl': {'kind': 'tmp',
1176                                                "capacity": 1073741824 }
1177                         },
1178                         'state': 'Committed',
1179                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1180                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1181                         'output_path': '/var/spool/cwl',
1182                         'output_ttl': 0,
1183                         'container_image': '99999999999999999999999999999993+99',
1184                         'command': ['nvidia-smi'],
1185                         'cwd': '/var/spool/cwl',
1186                         'scheduling_parameters': {},
1187                         'properties': {'cwl_input': {}},
1188                         'secret_mounts': {},
1189                         'output_storage_classes': ["default"]
1190                     }))
1191
1192
1193     # The test passes no builder.resources
1194     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1195     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1197     def test_match_local_docker(self, keepdocker, determine_image_id):
1198         arvados_cwl.add_arv_hints()
1199         arv_docker_clear_cache()
1200
1201         runner = mock.MagicMock()
1202         runner.ignore_docker_for_reuse = False
1203         runner.intermediate_output_ttl = 0
1204         runner.secret_store = cwltool.secrets.SecretStore()
1205         runner.api._rootDesc = {"revision": "20210628"}
1206
1207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1208                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1209         determine_image_id.side_effect = lambda x: "123"
1210         def execute(uuid):
1211             ex = mock.MagicMock()
1212             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1213                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1214             ex.execute.return_value = lookup[uuid]
1215             return ex
1216         runner.api.collections().get.side_effect = execute
1217
1218         tool = cmap({
1219             "inputs": [],
1220             "outputs": [],
1221             "baseCommand": "echo",
1222             "arguments": [],
1223             "id": "",
1224             "cwlVersion": "v1.0",
1225             "class": "org.w3id.cwl.cwl.CommandLineTool"
1226         })
1227
1228         loadingContext, runtimeContext = self.helper(runner, True)
1229
1230         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1231         arvtool.formatgraph = None
1232
1233         container_request = {
1234             'environment': {
1235                 'HOME': '/var/spool/cwl',
1236                 'TMPDIR': '/tmp'
1237             },
1238             'name': 'test_run_True',
1239             'runtime_constraints': {
1240                 'vcpus': 1,
1241                 'ram': 1073741824,
1242             },
1243             'use_existing': True,
1244             'priority': 500,
1245             'mounts': {
1246                 '/tmp': {'kind': 'tmp',
1247                          "capacity": 1073741824
1248                          },
1249                 '/var/spool/cwl': {'kind': 'tmp',
1250                                    "capacity": 1073741824 }
1251             },
1252             'state': 'Committed',
1253             'output_name': 'Output from step test_run_True',
1254             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1255             'output_path': '/var/spool/cwl',
1256             'output_ttl': 0,
1257             'container_image': '99999999999999999999999999999994+99',
1258             'command': ['echo'],
1259             'cwd': '/var/spool/cwl',
1260             'scheduling_parameters': {},
1261             'properties': {'cwl_input': {}},
1262             'secret_mounts': {},
1263             'output_storage_classes': ["default"]
1264         }
1265
1266         runtimeContext.match_local_docker = False
1267         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1268             j.run(runtimeContext)
1269             runner.api.container_requests().create.assert_called_with(
1270                 body=JsonDiffMatcher(container_request))
1271
1272         arv_docker_clear_cache()
1273         runtimeContext.match_local_docker = True
1274         container_request['container_image'] = '99999999999999999999999999999993+99'
1275         container_request['name'] = 'test_run_True_2'
1276         container_request['output_name'] = 'Output from step test_run_True_2'
1277         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1278             j.run(runtimeContext)
1279             runner.api.container_requests().create.assert_called_with(
1280                 body=JsonDiffMatcher(container_request))
1281
1282
1283     # The test passes no builder.resources
1284     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1285     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1286     def test_run_preemptible_hint(self, keepdocker):
1287         arvados_cwl.add_arv_hints()
1288         for enable_preemptible in (None, True, False):
1289             for preemptible_hint in (None, True, False):
1290                 arv_docker_clear_cache()
1291
1292                 runner = mock.MagicMock()
1293                 runner.ignore_docker_for_reuse = False
1294                 runner.intermediate_output_ttl = 0
1295                 runner.secret_store = cwltool.secrets.SecretStore()
1296                 runner.api._rootDesc = {"revision": "20210628"}
1297
1298                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1299                 runner.api.collections().get().execute.return_value = {
1300                     "portable_data_hash": "99999999999999999999999999999993+99"}
1301
1302                 if preemptible_hint is not None:
1303                     hints = [{
1304                         "class": "http://arvados.org/cwl#UsePreemptible",
1305                         "usePreemptible": preemptible_hint
1306                     }]
1307                 else:
1308                     hints = []
1309
1310                 tool = cmap({
1311                     "inputs": [],
1312                     "outputs": [],
1313                     "baseCommand": "ls",
1314                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1315                     "id": "",
1316                     "class": "CommandLineTool",
1317                     "cwlVersion": "v1.2",
1318                     "hints": hints
1319                 })
1320
1321                 loadingContext, runtimeContext = self.helper(runner)
1322
1323                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1324                 runtimeContext.enable_preemptible = enable_preemptible
1325
1326                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1327                 arvtool.formatgraph = None
1328
1329                 # Test the interactions between --enable/disable-preemptible
1330                 # and UsePreemptible hint
1331
1332                 if enable_preemptible is None:
1333                     if preemptible_hint is None:
1334                         sched = {}
1335                     else:
1336                         sched = {'preemptible': preemptible_hint}
1337                 else:
1338                     if preemptible_hint is None:
1339                         sched = {'preemptible': enable_preemptible}
1340                     else:
1341                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1342
1343                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1344                     j.run(runtimeContext)
1345                     runner.api.container_requests().create.assert_called_with(
1346                         body=JsonDiffMatcher({
1347                             'environment': {
1348                                 'HOME': '/var/spool/cwl',
1349                                 'TMPDIR': '/tmp'
1350                             },
1351                             'name': runtimeContext.name,
1352                             'runtime_constraints': {
1353                                 'vcpus': 1,
1354                                 'ram': 268435456
1355                             },
1356                             'use_existing': True,
1357                             'priority': 500,
1358                             'mounts': {
1359                                 '/tmp': {'kind': 'tmp',
1360                                          "capacity": 1073741824
1361                                      },
1362                                 '/var/spool/cwl': {'kind': 'tmp',
1363                                                    "capacity": 1073741824 }
1364                             },
1365                             'state': 'Committed',
1366                             'output_name': 'Output from step '+runtimeContext.name,
1367                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1368                             'output_path': '/var/spool/cwl',
1369                             'output_ttl': 0,
1370                             'container_image': '99999999999999999999999999999993+99',
1371                             'command': ['ls', '/var/spool/cwl'],
1372                             'cwd': '/var/spool/cwl',
1373                             'scheduling_parameters': sched,
1374                             'properties': {'cwl_input': {}},
1375                             'secret_mounts': {},
1376                             'output_storage_classes': ["default"]
1377                         }))
1378
1379
1380     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1381     def test_output_properties(self, keepdocker):
1382         arvados_cwl.add_arv_hints()
1383         for rev in ["20210628", "20220510"]:
1384             runner = mock.MagicMock()
1385             runner.ignore_docker_for_reuse = False
1386             runner.intermediate_output_ttl = 0
1387             runner.secret_store = cwltool.secrets.SecretStore()
1388             runner.api._rootDesc = {"revision": rev}
1389
1390             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1391             runner.api.collections().get().execute.return_value = {
1392                 "portable_data_hash": "99999999999999999999999999999993+99"}
1393
1394             tool = cmap({
1395                 "inputs": [{
1396                     "id": "inp",
1397                     "type": "string"
1398                 }],
1399                 "outputs": [],
1400                 "baseCommand": "ls",
1401                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1402                 "id": "",
1403                 "cwlVersion": "v1.2",
1404                 "class": "CommandLineTool",
1405                 "hints": [
1406                     {
1407                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1408                         "outputProperties": {
1409                             "foo": "bar",
1410                             "baz": "$(inputs.inp)"
1411                         }
1412                     }
1413                 ]
1414             })
1415
1416             loadingContext, runtimeContext = self.helper(runner)
1417             runtimeContext.name = "test_timelimit"
1418
1419             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1420             arvtool.formatgraph = None
1421
1422             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1423                 j.run(runtimeContext)
1424
1425             _, kwargs = runner.api.container_requests().create.call_args
1426             if rev == "20220510":
1427                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1428             else:
1429                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1430
1431
1432 class TestWorkflow(unittest.TestCase):
1433     def setUp(self):
1434         cwltool.process._names = set()
1435         arv_docker_clear_cache()
1436
1437     def helper(self, runner, enable_reuse=True):
1438         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1439
1440         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1441                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1442
1443         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1444         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1445         document_loader.fetch_text = document_loader.fetcher.fetch_text
1446         document_loader.check_exists = document_loader.fetcher.check_exists
1447
1448         loadingContext = arvados_cwl.context.ArvLoadingContext(
1449             {"avsc_names": avsc_names,
1450              "basedir": "",
1451              "make_fs_access": make_fs_access,
1452              "loader": document_loader,
1453              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1454              "construct_tool_object": runner.arv_make_tool})
1455         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1456             {"work_api": "containers",
1457              "basedir": "",
1458              "name": "test_run_wf_"+str(enable_reuse),
1459              "make_fs_access": make_fs_access,
1460              "tmpdir": "/tmp",
1461              "enable_reuse": enable_reuse,
1462              "priority": 500})
1463
1464         return loadingContext, runtimeContext
1465
1466     # The test passes no builder.resources
1467     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1468     @mock.patch("arvados.collection.CollectionReader")
1469     @mock.patch("arvados.collection.Collection")
1470     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1471     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1472         arvados_cwl.add_arv_hints()
1473
1474         api = mock.MagicMock()
1475         api._rootDesc = get_rootDesc()
1476
1477         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1478         self.assertEqual(runner.work_api, 'containers')
1479
1480         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1481         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1482         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1483                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1484
1485         runner.api.containers().current().execute.return_value = {}
1486
1487         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1488         runner.ignore_docker_for_reuse = False
1489         runner.num_retries = 0
1490         runner.secret_store = cwltool.secrets.SecretStore()
1491
1492         loadingContext, runtimeContext = self.helper(runner)
1493         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1494
1495         mockcollectionreader().exists.return_value = True
1496
1497         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1498         metadata["cwlVersion"] = tool["cwlVersion"]
1499
1500         mockc = mock.MagicMock()
1501         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1502         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1503
1504         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1505         arvtool.formatgraph = None
1506         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1507
1508         next(it).run(runtimeContext)
1509         next(it).run(runtimeContext)
1510
1511         with open("tests/wf/scatter2_subwf.cwl") as f:
1512             subwf = StripYAMLComments(f.read()).rstrip()
1513
1514         runner.api.container_requests().create.assert_called_with(
1515             body=JsonDiffMatcher({
1516                 "command": [
1517                     "cwltool",
1518                     "--no-container",
1519                     "--move-outputs",
1520                     "--preserve-entire-environment",
1521                     "workflow.cwl",
1522                     "cwl.input.yml"
1523                 ],
1524                 "container_image": "99999999999999999999999999999993+99",
1525                 "cwd": "/var/spool/cwl",
1526                 "environment": {
1527                     "HOME": "/var/spool/cwl",
1528                     "TMPDIR": "/tmp"
1529                 },
1530                 "mounts": {
1531                     "/keep/99999999999999999999999999999999+118": {
1532                         "kind": "collection",
1533                         "portable_data_hash": "99999999999999999999999999999999+118"
1534                     },
1535                     "/tmp": {
1536                         "capacity": 1073741824,
1537                         "kind": "tmp"
1538                     },
1539                     "/var/spool/cwl": {
1540                         "capacity": 1073741824,
1541                         "kind": "tmp"
1542                     },
1543                     "/var/spool/cwl/cwl.input.yml": {
1544                         "kind": "collection",
1545                         "path": "cwl.input.yml",
1546                         "portable_data_hash": "99999999999999999999999999999996+99"
1547                     },
1548                     "/var/spool/cwl/workflow.cwl": {
1549                         "kind": "collection",
1550                         "path": "workflow.cwl",
1551                         "portable_data_hash": "99999999999999999999999999999996+99"
1552                     },
1553                     "stdout": {
1554                         "kind": "file",
1555                         "path": "/var/spool/cwl/cwl.output.json"
1556                     }
1557                 },
1558                 "name": "scatterstep",
1559                 "output_name": "Output from step scatterstep",
1560                 "output_path": "/var/spool/cwl",
1561                 "output_ttl": 0,
1562                 "priority": 500,
1563                 "properties": {'cwl_input': {
1564                         "fileblub": {
1565                             "basename": "token.txt",
1566                             "class": "File",
1567                             "dirname": "/keep/99999999999999999999999999999999+118",
1568                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1569                             "nameext": ".txt",
1570                             "nameroot": "token",
1571                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1572                             "size": 0
1573                         },
1574                         "sleeptime": 5
1575                 }},
1576                 "runtime_constraints": {
1577                     "ram": 1073741824,
1578                     "vcpus": 1
1579                 },
1580                 "scheduling_parameters": {},
1581                 "secret_mounts": {},
1582                 "state": "Committed",
1583                 "use_existing": True,
1584                 'output_storage_classes': ["default"]
1585             }))
1586         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1587         mockc.open().__enter__().write.assert_has_calls([mock.call(
1588 '''{
1589   "fileblub": {
1590     "basename": "token.txt",
1591     "class": "File",
1592     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1593     "size": 0
1594   },
1595   "sleeptime": 5
1596 }''')])
1597
1598     # The test passes no builder.resources
1599     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1600     @mock.patch("arvados.collection.CollectionReader")
1601     @mock.patch("arvados.collection.Collection")
1602     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1603     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1604         arvados_cwl.add_arv_hints()
1605
1606         api = mock.MagicMock()
1607         api._rootDesc = get_rootDesc()
1608
1609         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1610         self.assertEqual(runner.work_api, 'containers')
1611
1612         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1613         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1614                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1615         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1616                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1617
1618         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1619         runner.ignore_docker_for_reuse = False
1620         runner.num_retries = 0
1621         runner.secret_store = cwltool.secrets.SecretStore()
1622
1623         loadingContext, runtimeContext = self.helper(runner)
1624         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1625         loadingContext.do_update = True
1626         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1627
1628         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1629
1630         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1631         arvtool.formatgraph = None
1632         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1633
1634         next(it).run(runtimeContext)
1635         next(it).run(runtimeContext)
1636
1637         with open("tests/wf/echo-subwf.cwl") as f:
1638             subwf = StripYAMLComments(f.read())
1639
1640         runner.api.container_requests().create.assert_called_with(
1641             body=JsonDiffMatcher({
1642                 'output_ttl': 0,
1643                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1644                 'scheduling_parameters': {},
1645                 'name': u'echo-subwf',
1646                 'secret_mounts': {},
1647                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1648                 'properties': {'cwl_input': {}},
1649                 'priority': 500,
1650                 'mounts': {
1651                     '/var/spool/cwl/cwl.input.yml': {
1652                         'portable_data_hash': '99999999999999999999999999999996+99',
1653                         'kind': 'collection',
1654                         'path': 'cwl.input.yml'
1655                     },
1656                     '/var/spool/cwl/workflow.cwl': {
1657                         'portable_data_hash': '99999999999999999999999999999996+99',
1658                         'kind': 'collection',
1659                         'path': 'workflow.cwl'
1660                     },
1661                     'stdout': {
1662                         'path': '/var/spool/cwl/cwl.output.json',
1663                         'kind': 'file'
1664                     },
1665                     '/tmp': {
1666                         'kind': 'tmp',
1667                         'capacity': 1073741824
1668                     }, '/var/spool/cwl': {
1669                         'kind': 'tmp',
1670                         'capacity': 3221225472
1671                     }
1672                 },
1673                 'state': 'Committed',
1674                 'output_path': '/var/spool/cwl',
1675                 'container_image': '99999999999999999999999999999993+99',
1676                 'command': [
1677                     u'cwltool',
1678                     u'--no-container',
1679                     u'--move-outputs',
1680                     u'--preserve-entire-environment',
1681                     u'workflow.cwl',
1682                     u'cwl.input.yml'
1683                 ],
1684                 'use_existing': True,
1685                 'output_name': u'Output from step echo-subwf',
1686                 'cwd': '/var/spool/cwl',
1687                 'output_storage_classes': ["default"]
1688             }))
1689
1690     def test_default_work_api(self):
1691         arvados_cwl.add_arv_hints()
1692
1693         api = mock.MagicMock()
1694         api._rootDesc = copy.deepcopy(get_rootDesc())
1695         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1696         self.assertEqual(runner.work_api, 'containers')