sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {'cwl_input': {}},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
 205
 206         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 207         runner.api.collections().get().execute.return_value = {
 208             "portable_data_hash": "99999999999999999999999999999993+99"}
 209
 210         tool = cmap({
 211             "inputs": [],
 212             "outputs": [],
 213             "hints": [{
 214                 "class": "ResourceRequirement",
 215                 "coresMin": 3,
 216                 "ramMin": 3000,
 217                 "tmpdirMin": 4000,
 218                 "outdirMin": 5000
 219             }, {
 220                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 221                 "keep_cache": 512
 222             }, {
 223                 "class": "http://arvados.org/cwl#APIRequirement",
 224             }, {
 225                 "class": "http://arvados.org/cwl#PartitionRequirement",
 226                 "partition": "blurb"
 227             }, {
 228                 "class": "http://arvados.org/cwl#IntermediateOutput",
 229                 "outputTTL": 7200
 230             }, {
 231                 "class": "WorkReuse",
 232                 "enableReuse": False
 233             }],
 234             "baseCommand": "ls",
 235             "id": "",
 236             "class": "CommandLineTool",
 237             "cwlVersion": "v1.2"
 238         })
 239
 240         loadingContext, runtimeContext = self.helper(runner)
 241         runtimeContext.name = "test_resource_requirements"
 242
 243         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 244         arvtool.formatgraph = None
 245         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 246             j.run(runtimeContext)
 247
 248         call_args, call_kwargs = runner.api.container_requests().create.call_args
 249
 250         call_body_expected = {
 251             'environment': {
 252                 'HOME': '/var/spool/cwl',
 253                 'TMPDIR': '/tmp'
 254             },
 255             'name': 'test_resource_requirements',
 256             'runtime_constraints': {
 257                 'vcpus': 3,
 258                 'ram': 3145728000,
 259                 'keep_cache_ram': 536870912,
 260                 'API': True
 261             },
 262             'use_existing': False,
 263             'priority': 500,
 264             'mounts': {
 265                 '/tmp': {'kind': 'tmp',
 266                          "capacity": 4194304000 },
 267                 '/var/spool/cwl': {'kind': 'tmp',
 268                                    "capacity": 5242880000 }
 269             },
 270             'state': 'Committed',
 271             'output_name': 'Output from step test_resource_requirements',
 272             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 273             'output_path': '/var/spool/cwl',
 274             'output_ttl': 7200,
 275             'container_image': '99999999999999999999999999999993+99',
 276             'command': ['ls'],
 277             'cwd': '/var/spool/cwl',
 278             'scheduling_parameters': {
 279                 'partitions': ['blurb']
 280             },
 281             'properties': {'cwl_input': {}},
 282             'secret_mounts': {},
 283             'output_storage_classes': ["default"]
 284         }
 285
 286         call_body = call_kwargs.get('body', None)
 287         self.assertNotEqual(None, call_body)
 288         for key in call_body:
 289             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 290
 291
 292     # The test passes some fields in builder.resources
 293     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 294     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 295     @mock.patch("arvados.collection.Collection")
 296     def test_initial_work_dir(self, collection_mock, keepdocker):
 297         runner = mock.MagicMock()
 298         runner.ignore_docker_for_reuse = False
 299         runner.intermediate_output_ttl = 0
 300         runner.secret_store = cwltool.secrets.SecretStore()
 301         runner.api._rootDesc = {"revision": "20210628"}
 302
 303         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 304         runner.api.collections().get().execute.return_value = {
 305             "portable_data_hash": "99999999999999999999999999999993+99"}
 306
 307         sourcemock = mock.MagicMock()
 308         def get_collection_mock(p):
 309             if "/" in p:
 310                 return (sourcemock, p.split("/", 1)[1])
 311             else:
 312                 return (sourcemock, "")
 313         runner.fs_access.get_collection.side_effect = get_collection_mock
 314
 315         vwdmock = mock.MagicMock()
 316         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 317
 318         tool = cmap({
 319             "inputs": [],
 320             "outputs": [],
 321             "hints": [{
 322                 "class": "InitialWorkDirRequirement",
 323                 "listing": [{
 324                     "class": "File",
 325                     "basename": "foo",
 326                     "location": "keep:99999999999999999999999999999995+99/bar"
 327                 },
 328                 {
 329                     "class": "Directory",
 330                     "basename": "foo2",
 331                     "location": "keep:99999999999999999999999999999995+99"
 332                 },
 333                 {
 334                     "class": "File",
 335                     "basename": "filename",
 336                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 337                 },
 338                 {
 339                     "class": "Directory",
 340                     "basename": "subdir",
 341                     "location": "keep:99999999999999999999999999999995+99/subdir"
 342                 }                        ]
 343             }],
 344             "baseCommand": "ls",
 345             "class": "CommandLineTool",
 346             "cwlVersion": "v1.2",
 347             "id": ""
 348         })
 349
 350         loadingContext, runtimeContext = self.helper(runner)
 351         runtimeContext.name = "test_initial_work_dir"
 352
 353         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 354
 355         arvtool.formatgraph = None
 356         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 357             j.run(runtimeContext)
 358
 359         call_args, call_kwargs = runner.api.container_requests().create.call_args
 360
 361         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 365
 366         call_body_expected = {
 367             'environment': {
 368                 'HOME': '/var/spool/cwl',
 369                 'TMPDIR': '/tmp'
 370             },
 371             'name': 'test_initial_work_dir',
 372             'runtime_constraints': {
 373                 'vcpus': 1,
 374                 'ram': 268435456
 375             },
 376             'use_existing': True,
 377             'priority': 500,
 378             'mounts': {
 379                 '/tmp': {'kind': 'tmp',
 380                          "capacity": 1073741824 },
 381                 '/var/spool/cwl': {'kind': 'tmp',
 382                                    "capacity": 1073741824 },
 383                 '/var/spool/cwl/foo': {
 384                     'kind': 'collection',
 385                     'path': 'foo',
 386                     'portable_data_hash': '99999999999999999999999999999996+99'
 387                 },
 388                 '/var/spool/cwl/foo2': {
 389                     'kind': 'collection',
 390                     'path': 'foo2',
 391                     'portable_data_hash': '99999999999999999999999999999996+99'
 392                 },
 393                 '/var/spool/cwl/filename': {
 394                     'kind': 'collection',
 395                     'path': 'filename',
 396                     'portable_data_hash': '99999999999999999999999999999996+99'
 397                 },
 398                 '/var/spool/cwl/subdir': {
 399                     'kind': 'collection',
 400                     'path': 'subdir',
 401                     'portable_data_hash': '99999999999999999999999999999996+99'
 402                 }
 403             },
 404             'state': 'Committed',
 405             'output_name': 'Output from step test_initial_work_dir',
 406             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 407             'output_path': '/var/spool/cwl',
 408             'output_ttl': 0,
 409             'container_image': '99999999999999999999999999999993+99',
 410             'command': ['ls'],
 411             'cwd': '/var/spool/cwl',
 412             'scheduling_parameters': {
 413             },
 414             'properties': {'cwl_input': {}},
 415             'secret_mounts': {},
 416             'output_storage_classes': ["default"]
 417         }
 418
 419         call_body = call_kwargs.get('body', None)
 420         self.assertNotEqual(None, call_body)
 421         for key in call_body:
 422             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 423
 424
 425     # Test redirecting stdin/stdout/stderr
 426     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 427     def test_redirects(self, keepdocker):
 428         runner = mock.MagicMock()
 429         runner.ignore_docker_for_reuse = False
 430         runner.intermediate_output_ttl = 0
 431         runner.secret_store = cwltool.secrets.SecretStore()
 432         runner.api._rootDesc = {"revision": "20210628"}
 433
 434         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 435         runner.api.collections().get().execute.return_value = {
 436             "portable_data_hash": "99999999999999999999999999999993+99"}
 437
 438         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 439
 440         tool = cmap({
 441             "inputs": [],
 442             "outputs": [],
 443             "baseCommand": "ls",
 444             "stdout": "stdout.txt",
 445             "stderr": "stderr.txt",
 446             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 447             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 448             "id": "",
 449             "class": "CommandLineTool",
 450             "cwlVersion": "v1.2"
 451         })
 452
 453         loadingContext, runtimeContext = self.helper(runner)
 454         runtimeContext.name = "test_run_redirect"
 455
 456         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 457         arvtool.formatgraph = None
 458         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 459             j.run(runtimeContext)
 460             runner.api.container_requests().create.assert_called_with(
 461                 body=JsonDiffMatcher({
 462                     'environment': {
 463                         'HOME': '/var/spool/cwl',
 464                         'TMPDIR': '/tmp'
 465                     },
 466                     'name': 'test_run_redirect',
 467                     'runtime_constraints': {
 468                         'vcpus': 1,
 469                         'ram': 268435456
 470                     },
 471                     'use_existing': True,
 472                     'priority': 500,
 473                     'mounts': {
 474                         '/tmp': {'kind': 'tmp',
 475                                  "capacity": 1073741824 },
 476                         '/var/spool/cwl': {'kind': 'tmp',
 477                                            "capacity": 1073741824 },
 478                         "stderr": {
 479                             "kind": "file",
 480                             "path": "/var/spool/cwl/stderr.txt"
 481                         },
 482                         "stdin": {
 483                             "kind": "collection",
 484                             "path": "file.txt",
 485                             "portable_data_hash": "99999999999999999999999999999996+99"
 486                         },
 487                         "stdout": {
 488                             "kind": "file",
 489                             "path": "/var/spool/cwl/stdout.txt"
 490                         },
 491                     },
 492                     'state': 'Committed',
 493                     "output_name": "Output from step test_run_redirect",
 494                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 495                     'output_path': '/var/spool/cwl',
 496                     'output_ttl': 0,
 497                     'container_image': '99999999999999999999999999999993+99',
 498                     'command': ['ls', '/var/spool/cwl'],
 499                     'cwd': '/var/spool/cwl',
 500                     'scheduling_parameters': {},
 501                     'properties': {'cwl_input': {}},
 502                     'secret_mounts': {},
 503                     'output_storage_classes': ["default"]
 504                 }))
 505
 506     @mock.patch("arvados.collection.Collection")
 507     def test_done(self, col):
 508         api = mock.MagicMock()
 509
 510         runner = mock.MagicMock()
 511         runner.api = api
 512         runner.num_retries = 0
 513         runner.ignore_docker_for_reuse = False
 514         runner.intermediate_output_ttl = 0
 515         runner.secret_store = cwltool.secrets.SecretStore()
 516
 517         runner.api.containers().get().execute.return_value = {"state":"Complete",
 518                                                               "output": "abc+123",
 519                                                               "exit_code": 0}
 520
 521         col().open.return_value = []
 522
 523         loadingContext, runtimeContext = self.helper(runner)
 524
 525         arvjob = arvados_cwl.ArvadosContainer(runner,
 526                                               runtimeContext,
 527                                               mock.MagicMock(),
 528                                               {},
 529                                               None,
 530                                               [],
 531                                               [],
 532                                               "testjob")
 533         arvjob.output_callback = mock.MagicMock()
 534         arvjob.collect_outputs = mock.MagicMock()
 535         arvjob.successCodes = [0]
 536         arvjob.outdir = "/var/spool/cwl"
 537         arvjob.output_ttl = 3600
 538         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 539
 540         arvjob.collect_outputs.return_value = {"out": "stuff"}
 541
 542         arvjob.done({
 543             "state": "Final",
 544             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 545             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 546             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 547             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 548             "modified_at": "2017-05-26T12:01:22Z",
 549             "properties": {}
 550         })
 551
 552         self.assertFalse(api.collections().create.called)
 553         self.assertFalse(runner.runtime_status_error.called)
 554
 555         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 556         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 557         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 558
 559         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 560                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 561
 562
 563     # Test to make sure we dont call runtime_status_update if we already did
 564     # some where higher up in the call stack
 565     @mock.patch("arvados_cwl.util.get_current_container")
 566     def test_recursive_runtime_status_update(self, gcc_mock):
 567         self.setup_and_test_container_executor_and_logging(gcc_mock)
 568         root_logger = logging.getLogger('')
 569
 570         # get_current_container is invoked when we call runtime_status_update
 571         # so try and log again!
 572         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 573         try:
 574             root_logger.error("First Error")
 575         except RuntimeError:
 576             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 577
 578
 579     # Test to make sure that an exception raised from
 580     # get_current_container doesn't cause the logger to raise an
 581     # exception
 582     @mock.patch("arvados_cwl.util.get_current_container")
 583     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 584         self.setup_and_test_container_executor_and_logging(gcc_mock)
 585         root_logger = logging.getLogger('')
 586
 587         # get_current_container is invoked when we call
 588         # runtime_status_update, it is going to also raise an
 589         # exception.
 590         gcc_mock.side_effect = Exception("Second Error")
 591         try:
 592             root_logger.error("First Error")
 593         except Exception:
 594             self.fail("Exception in logger should not propagate")
 595         self.assertTrue(gcc_mock.called)
 596
 597     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 598     @mock.patch("arvados_cwl.util.get_current_container")
 599     @mock.patch("arvados.collection.CollectionReader")
 600     @mock.patch("arvados.collection.Collection")
 601     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 602         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 603
 604         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 605         self.assertTrue(gcc_mock.called)
 606
 607         runner.num_retries = 0
 608         runner.ignore_docker_for_reuse = False
 609         runner.intermediate_output_ttl = 0
 610         runner.secret_store = cwltool.secrets.SecretStore()
 611         runner.label = mock.MagicMock()
 612         runner.label.return_value = '[container testjob]'
 613
 614         runner.api.containers().get().execute.return_value = {
 615             "state":"Complete",
 616             "output": "abc+123",
 617             "exit_code": 1,
 618             "log": "def+234"
 619         }
 620
 621         col().open.return_value = []
 622
 623         loadingContext, runtimeContext = self.helper(runner)
 624
 625         arvjob = arvados_cwl.ArvadosContainer(runner,
 626                                               runtimeContext,
 627                                               mock.MagicMock(),
 628                                               {},
 629                                               None,
 630                                               [],
 631                                               [],
 632                                               "testjob")
 633         arvjob.output_callback = mock.MagicMock()
 634         arvjob.collect_outputs = mock.MagicMock()
 635         arvjob.successCodes = [0]
 636         arvjob.outdir = "/var/spool/cwl"
 637         arvjob.output_ttl = 3600
 638         arvjob.collect_outputs.return_value = {"out": "stuff"}
 639
 640         arvjob.done({
 641             "state": "Final",
 642             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 643             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 644             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 645             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 646             "modified_at": "2017-05-26T12:01:22Z",
 647             "properties": {}
 648         })
 649
 650         rts_mock.assert_called_with(
 651             'error',
 652             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 653             '  ** log is empty **'
 654         )
 655         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 656
 657     # The test passes no builder.resources
 658     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 659     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 660     def test_mounts(self, keepdocker):
 661         runner = mock.MagicMock()
 662         runner.ignore_docker_for_reuse = False
 663         runner.intermediate_output_ttl = 0
 664         runner.secret_store = cwltool.secrets.SecretStore()
 665         runner.api._rootDesc = {"revision": "20210628"}
 666
 667         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 668         runner.api.collections().get().execute.return_value = {
 669             "portable_data_hash": "99999999999999999999999999999994+99",
 670             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 671
 672         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 673
 674         tool = cmap({
 675             "inputs": [
 676                 {"id": "p1",
 677                  "type": "Directory"}
 678             ],
 679             "outputs": [],
 680             "baseCommand": "ls",
 681             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 682             "id": "",
 683             "class": "CommandLineTool",
 684             "cwlVersion": "v1.2"
 685         })
 686
 687         loadingContext, runtimeContext = self.helper(runner)
 688         runtimeContext.name = "test_run_mounts"
 689
 690         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 691         arvtool.formatgraph = None
 692         job_order = {
 693             "p1": {
 694                 "class": "Directory",
 695                 "location": "keep:99999999999999999999999999999994+44",
 696                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 697                 "listing": [
 698                     {
 699                         "class": "File",
 700                         "location": "keep:99999999999999999999999999999994+44/file1",
 701                     },
 702                     {
 703                         "class": "File",
 704                         "location": "keep:99999999999999999999999999999994+44/file2",
 705                     }
 706                 ]
 707             }
 708         }
 709         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 710             j.run(runtimeContext)
 711             runner.api.container_requests().create.assert_called_with(
 712                 body=JsonDiffMatcher({
 713                     'environment': {
 714                         'HOME': '/var/spool/cwl',
 715                         'TMPDIR': '/tmp'
 716                     },
 717                     'name': 'test_run_mounts',
 718                     'runtime_constraints': {
 719                         'vcpus': 1,
 720                         'ram': 268435456
 721                     },
 722                     'use_existing': True,
 723                     'priority': 500,
 724                     'mounts': {
 725                         "/keep/99999999999999999999999999999994+44": {
 726                             "kind": "collection",
 727                             "portable_data_hash": "99999999999999999999999999999994+44",
 728                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 729                         },
 730                         '/tmp': {'kind': 'tmp',
 731                                  "capacity": 1073741824 },
 732                         '/var/spool/cwl': {'kind': 'tmp',
 733                                            "capacity": 1073741824 }
 734                     },
 735                     'state': 'Committed',
 736                     'output_name': 'Output from step test_run_mounts',
 737                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 738                     'output_path': '/var/spool/cwl',
 739                     'output_ttl': 0,
 740                     'container_image': '99999999999999999999999999999994+99',
 741                     'command': ['ls', '/var/spool/cwl'],
 742                     'cwd': '/var/spool/cwl',
 743                     'scheduling_parameters': {},
 744                     'properties': {'cwl_input': {
 745                         "p1": {
 746                             "basename": "99999999999999999999999999999994+44",
 747                             "class": "Directory",
 748                             "dirname": "/keep",
 749                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 750                             "listing": [
 751                                 {
 752                                     "basename": "file1",
 753                                     "class": "File",
 754                                     "dirname": "/keep/99999999999999999999999999999994+44",
 755                                     "location": "keep:99999999999999999999999999999994+44/file1",
 756                                     "nameext": "",
 757                                     "nameroot": "file1",
 758                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 759                                     "size": 0
 760                                 },
 761                                 {
 762                                     "basename": "file2",
 763                                     "class": "File",
 764                                     "dirname": "/keep/99999999999999999999999999999994+44",
 765                                     "location": "keep:99999999999999999999999999999994+44/file2",
 766                                     "nameext": "",
 767                                     "nameroot": "file2",
 768                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 769                                     "size": 0
 770                                 }
 771                             ],
 772                             "location": "keep:99999999999999999999999999999994+44",
 773                             "path": "/keep/99999999999999999999999999999994+44"
 774                         }
 775                     }},
 776                     'secret_mounts': {},
 777                     'output_storage_classes': ["default"]
 778                 }))
 779
 780     # The test passes no builder.resources
 781     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 782     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 783     def test_secrets(self, keepdocker):
 784         arvados_cwl.add_arv_hints()
 785         runner = mock.MagicMock()
 786         runner.ignore_docker_for_reuse = False
 787         runner.intermediate_output_ttl = 0
 788         runner.secret_store = cwltool.secrets.SecretStore()
 789         runner.api._rootDesc = {"revision": "20210628"}
 790
 791         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 792         runner.api.collections().get().execute.return_value = {
 793             "portable_data_hash": "99999999999999999999999999999993+99"}
 794
 795         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 796
 797         tool = cmap({"arguments": ["md5sum", "example.conf"],
 798                      "class": "CommandLineTool",
 799                      "cwlVersion": "v1.2",
 800                      "hints": [
 801                          {
 802                              "class": "http://commonwl.org/cwltool#Secrets",
 803                              "secrets": [
 804                                  "#secret_job.cwl/pw"
 805                              ]
 806                          }
 807                      ],
 808                      "id": "",
 809                      "inputs": [
 810                          {
 811                              "id": "#secret_job.cwl/pw",
 812                              "type": "string"
 813                          }
 814                      ],
 815                      "outputs": [
 816                      ],
 817                      "requirements": [
 818                          {
 819                              "class": "InitialWorkDirRequirement",
 820                              "listing": [
 821                                  {
 822                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 823                                      "entryname": "example.conf"
 824                                  }
 825                              ]
 826                          }
 827                      ]})
 828
 829         loadingContext, runtimeContext = self.helper(runner)
 830         runtimeContext.name = "test_secrets"
 831
 832         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 833         arvtool.formatgraph = None
 834
 835         job_order = {"pw": "blorp"}
 836         runner.secret_store.store(["pw"], job_order)
 837
 838         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 839             j.run(runtimeContext)
 840             runner.api.container_requests().create.assert_called_with(
 841                 body=JsonDiffMatcher({
 842                     'environment': {
 843                         'HOME': '/var/spool/cwl',
 844                         'TMPDIR': '/tmp'
 845                     },
 846                     'name': 'test_secrets',
 847                     'runtime_constraints': {
 848                         'vcpus': 1,
 849                         'ram': 268435456
 850                     },
 851                     'use_existing': True,
 852                     'priority': 500,
 853                     'mounts': {
 854                         '/tmp': {'kind': 'tmp',
 855                                  "capacity": 1073741824
 856                              },
 857                         '/var/spool/cwl': {'kind': 'tmp',
 858                                            "capacity": 1073741824 }
 859                     },
 860                     'state': 'Committed',
 861                     'output_name': 'Output from step test_secrets',
 862                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 863                     'output_path': '/var/spool/cwl',
 864                     'output_ttl': 0,
 865                     'container_image': '99999999999999999999999999999993+99',
 866                     'command': ['md5sum', 'example.conf'],
 867                     'cwd': '/var/spool/cwl',
 868                     'scheduling_parameters': {},
 869                     'properties': {'cwl_input': job_order},
 870                     "secret_mounts": {
 871                         "/var/spool/cwl/example.conf": {
 872                             "content": "username: user\npassword: blorp\n",
 873                             "kind": "text"
 874                         }
 875                     },
 876                     'output_storage_classes': ["default"]
 877                 }))
 878
 879     # The test passes no builder.resources
 880     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 881     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 882     def test_timelimit(self, keepdocker):
 883         runner = mock.MagicMock()
 884         runner.ignore_docker_for_reuse = False
 885         runner.intermediate_output_ttl = 0
 886         runner.secret_store = cwltool.secrets.SecretStore()
 887         runner.api._rootDesc = {"revision": "20210628"}
 888
 889         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 890         runner.api.collections().get().execute.return_value = {
 891             "portable_data_hash": "99999999999999999999999999999993+99"}
 892
 893         tool = cmap({
 894             "inputs": [],
 895             "outputs": [],
 896             "baseCommand": "ls",
 897             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 898             "id": "",
 899             "cwlVersion": "v1.2",
 900             "class": "CommandLineTool",
 901             "hints": [
 902                 {
 903                     "class": "ToolTimeLimit",
 904                     "timelimit": 42
 905                 }
 906             ]
 907         })
 908
 909         loadingContext, runtimeContext = self.helper(runner)
 910         runtimeContext.name = "test_timelimit"
 911
 912         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 913         arvtool.formatgraph = None
 914
 915         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 916             j.run(runtimeContext)
 917
 918         _, kwargs = runner.api.container_requests().create.call_args
 919         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 920
 921
 922     # The test passes no builder.resources
 923     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 924     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 925     def test_setting_storage_class(self, keepdocker):
 926         arv_docker_clear_cache()
 927
 928         runner = mock.MagicMock()
 929         runner.ignore_docker_for_reuse = False
 930         runner.intermediate_output_ttl = 0
 931         runner.secret_store = cwltool.secrets.SecretStore()
 932         runner.api._rootDesc = {"revision": "20210628"}
 933
 934         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 935         runner.api.collections().get().execute.return_value = {
 936             "portable_data_hash": "99999999999999999999999999999993+99"}
 937
 938         tool = cmap({
 939             "inputs": [],
 940             "outputs": [],
 941             "baseCommand": "ls",
 942             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 943             "id": "",
 944             "cwlVersion": "v1.2",
 945             "class": "CommandLineTool",
 946             "hints": [
 947                 {
 948                     "class": "http://arvados.org/cwl#OutputStorageClass",
 949                     "finalStorageClass": ["baz_sc", "qux_sc"],
 950                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 951                 }
 952             ]
 953         })
 954
 955         loadingContext, runtimeContext = self.helper(runner, True)
 956
 957         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 958         arvtool.formatgraph = None
 959
 960         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 961             j.run(runtimeContext)
 962             runner.api.container_requests().create.assert_called_with(
 963                 body=JsonDiffMatcher({
 964                     'environment': {
 965                         'HOME': '/var/spool/cwl',
 966                         'TMPDIR': '/tmp'
 967                     },
 968                     'name': 'test_run_True',
 969                     'runtime_constraints': {
 970                         'vcpus': 1,
 971                         'ram': 268435456
 972                     },
 973                     'use_existing': True,
 974                     'priority': 500,
 975                     'mounts': {
 976                         '/tmp': {'kind': 'tmp',
 977                                  "capacity": 1073741824
 978                              },
 979                         '/var/spool/cwl': {'kind': 'tmp',
 980                                            "capacity": 1073741824 }
 981                     },
 982                     'state': 'Committed',
 983                     'output_name': 'Output from step test_run_True',
 984                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 985                     'output_path': '/var/spool/cwl',
 986                     'output_ttl': 0,
 987                     'container_image': '99999999999999999999999999999993+99',
 988                     'command': ['ls', '/var/spool/cwl'],
 989                     'cwd': '/var/spool/cwl',
 990                     'scheduling_parameters': {},
 991                     'properties': {'cwl_input': {}},
 992                     'secret_mounts': {},
 993                     'output_storage_classes': ["foo_sc", "bar_sc"]
 994                 }))
 995
 996
 997     # The test passes no builder.resources
 998     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 999     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1000     def test_setting_process_properties(self, keepdocker):
1001         arv_docker_clear_cache()
1002
1003         runner = mock.MagicMock()
1004         runner.ignore_docker_for_reuse = False
1005         runner.intermediate_output_ttl = 0
1006         runner.secret_store = cwltool.secrets.SecretStore()
1007         runner.api._rootDesc = {"revision": "20210628"}
1008
1009         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1010         runner.api.collections().get().execute.return_value = {
1011             "portable_data_hash": "99999999999999999999999999999993+99"}
1012
1013         tool = cmap({
1014             "inputs": [
1015                 {"id": "x", "type": "string"}],
1016             "outputs": [],
1017             "baseCommand": "ls",
1018             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1019             "id": "",
1020             "class": "CommandLineTool",
1021             "cwlVersion": "v1.2",
1022             "hints": [
1023             {
1024                 "class": "http://arvados.org/cwl#ProcessProperties",
1025                 "processProperties": [
1026                     {"propertyName": "foo",
1027                      "propertyValue": "bar"},
1028                     {"propertyName": "baz",
1029                      "propertyValue": "$(inputs.x)"},
1030                     {"propertyName": "quux",
1031                      "propertyValue": {
1032                          "q1": 1,
1033                          "q2": 2
1034                      }
1035                     }
1036                 ],
1037             }
1038         ]
1039         })
1040
1041         loadingContext, runtimeContext = self.helper(runner, True)
1042
1043         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1044         arvtool.formatgraph = None
1045
1046         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1047             j.run(runtimeContext)
1048             runner.api.container_requests().create.assert_called_with(
1049                 body=JsonDiffMatcher({
1050                     'environment': {
1051                         'HOME': '/var/spool/cwl',
1052                         'TMPDIR': '/tmp'
1053                     },
1054                     'name': 'test_run_True',
1055                     'runtime_constraints': {
1056                         'vcpus': 1,
1057                         'ram': 268435456
1058                     },
1059                     'use_existing': True,
1060                     'priority': 500,
1061                     'mounts': {
1062                         '/tmp': {'kind': 'tmp',
1063                                  "capacity": 1073741824
1064                              },
1065                         '/var/spool/cwl': {'kind': 'tmp',
1066                                            "capacity": 1073741824 }
1067                     },
1068                     'state': 'Committed',
1069                     'output_name': 'Output from step test_run_True',
1070                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1071                     'output_path': '/var/spool/cwl',
1072                     'output_ttl': 0,
1073                     'container_image': '99999999999999999999999999999993+99',
1074                     'command': ['ls', '/var/spool/cwl'],
1075                     'cwd': '/var/spool/cwl',
1076                     'scheduling_parameters': {},
1077                     'properties': {
1078                         "baz": "blorp",
1079                         "cwl_input": {"x": "blorp"},
1080                         "foo": "bar",
1081                         "quux": {
1082                             "q1": 1,
1083                             "q2": 2
1084                         }
1085                     },
1086                     'secret_mounts': {},
1087                     'output_storage_classes': ["default"]
1088                 }))
1089
1090
1091     # The test passes no builder.resources
1092     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1093     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1094     def test_cuda_requirement(self, keepdocker):
1095         arvados_cwl.add_arv_hints()
1096         arv_docker_clear_cache()
1097
1098         runner = mock.MagicMock()
1099         runner.ignore_docker_for_reuse = False
1100         runner.intermediate_output_ttl = 0
1101         runner.secret_store = cwltool.secrets.SecretStore()
1102         runner.api._rootDesc = {"revision": "20210628"}
1103
1104         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1105         runner.api.collections().get().execute.return_value = {
1106             "portable_data_hash": "99999999999999999999999999999993+99"}
1107
1108         test_cwl_req = [{
1109                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1110                 "cudaVersionMin": "11.0",
1111                 "cudaComputeCapability": "9.0",
1112             }, {
1113                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1114                 "cudaVersionMin": "11.0",
1115                 "cudaComputeCapability": "9.0",
1116                 "cudaDeviceCountMin": 2
1117             }, {
1118                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1119                 "cudaVersionMin": "11.0",
1120                 "cudaComputeCapability": ["4.0", "5.0"],
1121                 "cudaDeviceCountMin": 2
1122             }]
1123
1124         test_arv_req = [{
1125             'device_count': 1,
1126             'driver_version': "11.0",
1127             'hardware_capability': "9.0"
1128         }, {
1129             'device_count': 2,
1130             'driver_version': "11.0",
1131             'hardware_capability': "9.0"
1132         }, {
1133             'device_count': 2,
1134             'driver_version': "11.0",
1135             'hardware_capability': "4.0"
1136         }]
1137
1138         for test_case in range(0, len(test_cwl_req)):
1139
1140             tool = cmap({
1141                 "inputs": [],
1142                 "outputs": [],
1143                 "baseCommand": "nvidia-smi",
1144                 "arguments": [],
1145                 "id": "",
1146                 "cwlVersion": "v1.2",
1147                 "class": "CommandLineTool",
1148                 "requirements": [test_cwl_req[test_case]]
1149             })
1150
1151             loadingContext, runtimeContext = self.helper(runner, True)
1152
1153             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1154             arvtool.formatgraph = None
1155
1156             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1157                 j.run(runtimeContext)
1158                 runner.api.container_requests().create.assert_called_with(
1159                     body=JsonDiffMatcher({
1160                         'environment': {
1161                             'HOME': '/var/spool/cwl',
1162                             'TMPDIR': '/tmp'
1163                         },
1164                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1165                         'runtime_constraints': {
1166                             'vcpus': 1,
1167                             'ram': 268435456,
1168                             'cuda': test_arv_req[test_case]
1169                         },
1170                         'use_existing': True,
1171                         'priority': 500,
1172                         'mounts': {
1173                             '/tmp': {'kind': 'tmp',
1174                                      "capacity": 1073741824
1175                                  },
1176                             '/var/spool/cwl': {'kind': 'tmp',
1177                                                "capacity": 1073741824 }
1178                         },
1179                         'state': 'Committed',
1180                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1181                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1182                         'output_path': '/var/spool/cwl',
1183                         'output_ttl': 0,
1184                         'container_image': '99999999999999999999999999999993+99',
1185                         'command': ['nvidia-smi'],
1186                         'cwd': '/var/spool/cwl',
1187                         'scheduling_parameters': {},
1188                         'properties': {'cwl_input': {}},
1189                         'secret_mounts': {},
1190                         'output_storage_classes': ["default"]
1191                     }))
1192
1193
1194     # The test passes no builder.resources
1195     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1196     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1198     def test_match_local_docker(self, keepdocker, determine_image_id):
1199         arvados_cwl.add_arv_hints()
1200         arv_docker_clear_cache()
1201
1202         runner = mock.MagicMock()
1203         runner.ignore_docker_for_reuse = False
1204         runner.intermediate_output_ttl = 0
1205         runner.secret_store = cwltool.secrets.SecretStore()
1206         runner.api._rootDesc = {"revision": "20210628"}
1207
1208         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1209                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1210         determine_image_id.side_effect = lambda x: "123"
1211         def execute(uuid):
1212             ex = mock.MagicMock()
1213             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1214                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1215             ex.execute.return_value = lookup[uuid]
1216             return ex
1217         runner.api.collections().get.side_effect = execute
1218
1219         tool = cmap({
1220             "inputs": [],
1221             "outputs": [],
1222             "baseCommand": "echo",
1223             "arguments": [],
1224             "id": "",
1225             "cwlVersion": "v1.0",
1226             "class": "org.w3id.cwl.cwl.CommandLineTool"
1227         })
1228
1229         loadingContext, runtimeContext = self.helper(runner, True)
1230
1231         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1232         arvtool.formatgraph = None
1233
1234         container_request = {
1235             'environment': {
1236                 'HOME': '/var/spool/cwl',
1237                 'TMPDIR': '/tmp'
1238             },
1239             'name': 'test_run_True',
1240             'runtime_constraints': {
1241                 'vcpus': 1,
1242                 'ram': 1073741824,
1243             },
1244             'use_existing': True,
1245             'priority': 500,
1246             'mounts': {
1247                 '/tmp': {'kind': 'tmp',
1248                          "capacity": 1073741824
1249                          },
1250                 '/var/spool/cwl': {'kind': 'tmp',
1251                                    "capacity": 1073741824 }
1252             },
1253             'state': 'Committed',
1254             'output_name': 'Output from step test_run_True',
1255             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1256             'output_path': '/var/spool/cwl',
1257             'output_ttl': 0,
1258             'container_image': '99999999999999999999999999999994+99',
1259             'command': ['echo'],
1260             'cwd': '/var/spool/cwl',
1261             'scheduling_parameters': {},
1262             'properties': {'cwl_input': {}},
1263             'secret_mounts': {},
1264             'output_storage_classes': ["default"]
1265         }
1266
1267         runtimeContext.match_local_docker = False
1268         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1269             j.run(runtimeContext)
1270             runner.api.container_requests().create.assert_called_with(
1271                 body=JsonDiffMatcher(container_request))
1272
1273         arv_docker_clear_cache()
1274         runtimeContext.match_local_docker = True
1275         container_request['container_image'] = '99999999999999999999999999999993+99'
1276         container_request['name'] = 'test_run_True_2'
1277         container_request['output_name'] = 'Output from step test_run_True_2'
1278         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1279             j.run(runtimeContext)
1280             runner.api.container_requests().create.assert_called_with(
1281                 body=JsonDiffMatcher(container_request))
1282
1283
1284     # The test passes no builder.resources
1285     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1286     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1287     def test_run_preemptible_hint(self, keepdocker):
1288         arvados_cwl.add_arv_hints()
1289         for enable_preemptible in (None, True, False):
1290             for preemptible_hint in (None, True, False):
1291                 arv_docker_clear_cache()
1292
1293                 runner = mock.MagicMock()
1294                 runner.ignore_docker_for_reuse = False
1295                 runner.intermediate_output_ttl = 0
1296                 runner.secret_store = cwltool.secrets.SecretStore()
1297                 runner.api._rootDesc = {"revision": "20210628"}
1298
1299                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1300                 runner.api.collections().get().execute.return_value = {
1301                     "portable_data_hash": "99999999999999999999999999999993+99"}
1302
1303                 if preemptible_hint is not None:
1304                     hints = [{
1305                         "class": "http://arvados.org/cwl#UsePreemptible",
1306                         "usePreemptible": preemptible_hint
1307                     }]
1308                 else:
1309                     hints = []
1310
1311                 tool = cmap({
1312                     "inputs": [],
1313                     "outputs": [],
1314                     "baseCommand": "ls",
1315                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1316                     "id": "",
1317                     "class": "CommandLineTool",
1318                     "cwlVersion": "v1.2",
1319                     "hints": hints
1320                 })
1321
1322                 loadingContext, runtimeContext = self.helper(runner)
1323
1324                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1325                 runtimeContext.enable_preemptible = enable_preemptible
1326
1327                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1328                 arvtool.formatgraph = None
1329
1330                 # Test the interactions between --enable/disable-preemptible
1331                 # and UsePreemptible hint
1332
1333                 if enable_preemptible is None:
1334                     if preemptible_hint is None:
1335                         sched = {}
1336                     else:
1337                         sched = {'preemptible': preemptible_hint}
1338                 else:
1339                     if preemptible_hint is None:
1340                         sched = {'preemptible': enable_preemptible}
1341                     else:
1342                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1343
1344                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1345                     j.run(runtimeContext)
1346                     runner.api.container_requests().create.assert_called_with(
1347                         body=JsonDiffMatcher({
1348                             'environment': {
1349                                 'HOME': '/var/spool/cwl',
1350                                 'TMPDIR': '/tmp'
1351                             },
1352                             'name': runtimeContext.name,
1353                             'runtime_constraints': {
1354                                 'vcpus': 1,
1355                                 'ram': 268435456
1356                             },
1357                             'use_existing': True,
1358                             'priority': 500,
1359                             'mounts': {
1360                                 '/tmp': {'kind': 'tmp',
1361                                          "capacity": 1073741824
1362                                      },
1363                                 '/var/spool/cwl': {'kind': 'tmp',
1364                                                    "capacity": 1073741824 }
1365                             },
1366                             'state': 'Committed',
1367                             'output_name': 'Output from step '+runtimeContext.name,
1368                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1369                             'output_path': '/var/spool/cwl',
1370                             'output_ttl': 0,
1371                             'container_image': '99999999999999999999999999999993+99',
1372                             'command': ['ls', '/var/spool/cwl'],
1373                             'cwd': '/var/spool/cwl',
1374                             'scheduling_parameters': sched,
1375                             'properties': {'cwl_input': {}},
1376                             'secret_mounts': {},
1377                             'output_storage_classes': ["default"]
1378                         }))
1379
1380
1381     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1382     def test_output_properties(self, keepdocker):
1383         arvados_cwl.add_arv_hints()
1384         for rev in ["20210628", "20220510"]:
1385             runner = mock.MagicMock()
1386             runner.ignore_docker_for_reuse = False
1387             runner.intermediate_output_ttl = 0
1388             runner.secret_store = cwltool.secrets.SecretStore()
1389             runner.api._rootDesc = {"revision": rev}
1390
1391             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1392             runner.api.collections().get().execute.return_value = {
1393                 "portable_data_hash": "99999999999999999999999999999993+99"}
1394
1395             tool = cmap({
1396                 "inputs": [{
1397                     "id": "inp",
1398                     "type": "string"
1399                 }],
1400                 "outputs": [],
1401                 "baseCommand": "ls",
1402                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1403                 "id": "",
1404                 "cwlVersion": "v1.2",
1405                 "class": "CommandLineTool",
1406                 "hints": [
1407                     {
1408                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1409                         "outputProperties": {
1410                             "foo": "bar",
1411                             "baz": "$(inputs.inp)"
1412                         }
1413                     }
1414                 ]
1415             })
1416
1417             loadingContext, runtimeContext = self.helper(runner)
1418             runtimeContext.name = "test_timelimit"
1419
1420             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1421             arvtool.formatgraph = None
1422
1423             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1424                 j.run(runtimeContext)
1425
1426             _, kwargs = runner.api.container_requests().create.call_args
1427             if rev == "20220510":
1428                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1429             else:
1430                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1431
1432
1433 class TestWorkflow(unittest.TestCase):
1434     def setUp(self):
1435         cwltool.process._names = set()
1436         arv_docker_clear_cache()
1437
1438     def helper(self, runner, enable_reuse=True):
1439         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1440
1441         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1442                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1443
1444         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1445         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1446         document_loader.fetch_text = document_loader.fetcher.fetch_text
1447         document_loader.check_exists = document_loader.fetcher.check_exists
1448
1449         loadingContext = arvados_cwl.context.ArvLoadingContext(
1450             {"avsc_names": avsc_names,
1451              "basedir": "",
1452              "make_fs_access": make_fs_access,
1453              "loader": document_loader,
1454              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1455              "construct_tool_object": runner.arv_make_tool})
1456         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1457             {"work_api": "containers",
1458              "basedir": "",
1459              "name": "test_run_wf_"+str(enable_reuse),
1460              "make_fs_access": make_fs_access,
1461              "tmpdir": "/tmp",
1462              "enable_reuse": enable_reuse,
1463              "priority": 500})
1464
1465         return loadingContext, runtimeContext
1466
1467     # The test passes no builder.resources
1468     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1469     @mock.patch("arvados.collection.CollectionReader")
1470     @mock.patch("arvados.collection.Collection")
1471     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1472     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1473         arvados_cwl.add_arv_hints()
1474
1475         api = mock.MagicMock()
1476         api._rootDesc = get_rootDesc()
1477
1478         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1479         self.assertEqual(runner.work_api, 'containers')
1480
1481         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1482         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1483         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1484                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1485
1486         runner.api.containers().current().execute.return_value = {}
1487
1488         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1489         runner.ignore_docker_for_reuse = False
1490         runner.num_retries = 0
1491         runner.secret_store = cwltool.secrets.SecretStore()
1492
1493         loadingContext, runtimeContext = self.helper(runner)
1494         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1495
1496         mockcollectionreader().exists.return_value = True
1497
1498         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1499         metadata["cwlVersion"] = tool["cwlVersion"]
1500
1501         mockc = mock.MagicMock()
1502         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1503         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1504
1505         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1506         arvtool.formatgraph = None
1507         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1508
1509         next(it).run(runtimeContext)
1510         next(it).run(runtimeContext)
1511
1512         with open("tests/wf/scatter2_subwf.cwl") as f:
1513             subwf = StripYAMLComments(f.read()).rstrip()
1514
1515         runner.api.container_requests().create.assert_called_with(
1516             body=JsonDiffMatcher({
1517                 "command": [
1518                     "cwltool",
1519                     "--no-container",
1520                     "--move-outputs",
1521                     "--preserve-entire-environment",
1522                     "workflow.cwl",
1523                     "cwl.input.yml"
1524                 ],
1525                 "container_image": "99999999999999999999999999999993+99",
1526                 "cwd": "/var/spool/cwl",
1527                 "environment": {
1528                     "HOME": "/var/spool/cwl",
1529                     "TMPDIR": "/tmp"
1530                 },
1531                 "mounts": {
1532                     "/keep/99999999999999999999999999999999+118": {
1533                         "kind": "collection",
1534                         "portable_data_hash": "99999999999999999999999999999999+118"
1535                     },
1536                     "/tmp": {
1537                         "capacity": 1073741824,
1538                         "kind": "tmp"
1539                     },
1540                     "/var/spool/cwl": {
1541                         "capacity": 1073741824,
1542                         "kind": "tmp"
1543                     },
1544                     "/var/spool/cwl/cwl.input.yml": {
1545                         "kind": "collection",
1546                         "path": "cwl.input.yml",
1547                         "portable_data_hash": "99999999999999999999999999999996+99"
1548                     },
1549                     "/var/spool/cwl/workflow.cwl": {
1550                         "kind": "collection",
1551                         "path": "workflow.cwl",
1552                         "portable_data_hash": "99999999999999999999999999999996+99"
1553                     },
1554                     "stdout": {
1555                         "kind": "file",
1556                         "path": "/var/spool/cwl/cwl.output.json"
1557                     }
1558                 },
1559                 "name": "scatterstep",
1560                 "output_name": "Output from step scatterstep",
1561                 "output_path": "/var/spool/cwl",
1562                 "output_ttl": 0,
1563                 "priority": 500,
1564                 "properties": {'cwl_input': {
1565                         "fileblub": {
1566                             "basename": "token.txt",
1567                             "class": "File",
1568                             "dirname": "/keep/99999999999999999999999999999999+118",
1569                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1570                             "nameext": ".txt",
1571                             "nameroot": "token",
1572                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1573                             "size": 0
1574                         },
1575                         "sleeptime": 5
1576                 }},
1577                 "runtime_constraints": {
1578                     "ram": 1073741824,
1579                     "vcpus": 1
1580                 },
1581                 "scheduling_parameters": {},
1582                 "secret_mounts": {},
1583                 "state": "Committed",
1584                 "use_existing": True,
1585                 'output_storage_classes': ["default"]
1586             }))
1587         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1588         mockc.open().__enter__().write.assert_has_calls([mock.call(
1589 '''{
1590   "fileblub": {
1591     "basename": "token.txt",
1592     "class": "File",
1593     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1594     "size": 0
1595   },
1596   "sleeptime": 5
1597 }''')])
1598
1599     # The test passes no builder.resources
1600     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1601     @mock.patch("arvados.collection.CollectionReader")
1602     @mock.patch("arvados.collection.Collection")
1603     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1604     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1605         arvados_cwl.add_arv_hints()
1606
1607         api = mock.MagicMock()
1608         api._rootDesc = get_rootDesc()
1609
1610         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1611         self.assertEqual(runner.work_api, 'containers')
1612
1613         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1614         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1615                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1616         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1617                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1618
1619         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1620         runner.ignore_docker_for_reuse = False
1621         runner.num_retries = 0
1622         runner.secret_store = cwltool.secrets.SecretStore()
1623
1624         loadingContext, runtimeContext = self.helper(runner)
1625         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1626         loadingContext.do_update = True
1627         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1628
1629         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1630
1631         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1632         arvtool.formatgraph = None
1633         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1634
1635         next(it).run(runtimeContext)
1636         next(it).run(runtimeContext)
1637
1638         with open("tests/wf/echo-subwf.cwl") as f:
1639             subwf = StripYAMLComments(f.read())
1640
1641         runner.api.container_requests().create.assert_called_with(
1642             body=JsonDiffMatcher({
1643                 'output_ttl': 0,
1644                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1645                 'scheduling_parameters': {},
1646                 'name': u'echo-subwf',
1647                 'secret_mounts': {},
1648                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1649                 'properties': {'cwl_input': {}},
1650                 'priority': 500,
1651                 'mounts': {
1652                     '/var/spool/cwl/cwl.input.yml': {
1653                         'portable_data_hash': '99999999999999999999999999999996+99',
1654                         'kind': 'collection',
1655                         'path': 'cwl.input.yml'
1656                     },
1657                     '/var/spool/cwl/workflow.cwl': {
1658                         'portable_data_hash': '99999999999999999999999999999996+99',
1659                         'kind': 'collection',
1660                         'path': 'workflow.cwl'
1661                     },
1662                     'stdout': {
1663                         'path': '/var/spool/cwl/cwl.output.json',
1664                         'kind': 'file'
1665                     },
1666                     '/tmp': {
1667                         'kind': 'tmp',
1668                         'capacity': 1073741824
1669                     }, '/var/spool/cwl': {
1670                         'kind': 'tmp',
1671                         'capacity': 3221225472
1672                     }
1673                 },
1674                 'state': 'Committed',
1675                 'output_path': '/var/spool/cwl',
1676                 'container_image': '99999999999999999999999999999993+99',
1677                 'command': [
1678                     u'cwltool',
1679                     u'--no-container',
1680                     u'--move-outputs',
1681                     u'--preserve-entire-environment',
1682                     u'workflow.cwl',
1683                     u'cwl.input.yml'
1684                 ],
1685                 'use_existing': True,
1686                 'output_name': u'Output from step echo-subwf',
1687                 'cwd': '/var/spool/cwl',
1688                 'output_storage_classes': ["default"]
1689             }))
1690
1691     def test_default_work_api(self):
1692         arvados_cwl.add_arv_hints()
1693
1694         api = mock.MagicMock()
1695         api._rootDesc = copy.deepcopy(get_rootDesc())
1696         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1697         self.assertEqual(runner.work_api, 'containers')