sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 139
 140             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 141             runner.api.collections().get().execute.return_value = {
 142                 "portable_data_hash": "99999999999999999999999999999993+99"}
 143
 144             tool = cmap({
 145                 "inputs": [],
 146                 "outputs": [],
 147                 "baseCommand": "ls",
 148                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 149                 "id": "",
 150                 "class": "CommandLineTool",
 151                 "cwlVersion": "v1.2"
 152             })
 153
 154             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 155
 156             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 157             arvtool.formatgraph = None
 158
 159             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 160                 j.run(runtimeContext)
 161                 runner.api.container_requests().create.assert_called_with(
 162                     body=JsonDiffMatcher({
 163                         'environment': {
 164                             'HOME': '/var/spool/cwl',
 165                             'TMPDIR': '/tmp'
 166                         },
 167                         'name': 'test_run_'+str(enable_reuse),
 168                         'runtime_constraints': {
 169                             'vcpus': 1,
 170                             'ram': 268435456
 171                         },
 172                         'use_existing': enable_reuse,
 173                         'priority': 500,
 174                         'mounts': {
 175                             '/tmp': {'kind': 'tmp',
 176                                      "capacity": 1073741824
 177                                  },
 178                             '/var/spool/cwl': {'kind': 'tmp',
 179                                                "capacity": 1073741824 }
 180                         },
 181                         'state': 'Committed',
 182                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 183                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                         'output_path': '/var/spool/cwl',
 185                         'output_ttl': 0,
 186                         'container_image': '99999999999999999999999999999993+99',
 187                         'command': ['ls', '/var/spool/cwl'],
 188                         'cwd': '/var/spool/cwl',
 189                         'scheduling_parameters': {},
 190                         'properties': {'cwl_input': {}},
 191                         'secret_mounts': {},
 192                         'output_storage_classes': ["default"]
 193                     }))
 194
 195     # The test passes some fields in builder.resources
 196     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 198     def test_resource_requirements(self, keepdocker):
 199         arvados_cwl.add_arv_hints()
 200         runner = mock.MagicMock()
 201         runner.ignore_docker_for_reuse = False
 202         runner.intermediate_output_ttl = 3600
 203         runner.secret_store = cwltool.secrets.SecretStore()
 204         runner.api._rootDesc = {"revision": "20210628"}
 205         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 206
 207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 208         runner.api.collections().get().execute.return_value = {
 209             "portable_data_hash": "99999999999999999999999999999993+99"}
 210
 211         tool = cmap({
 212             "inputs": [],
 213             "outputs": [],
 214             "hints": [{
 215                 "class": "ResourceRequirement",
 216                 "coresMin": 3,
 217                 "ramMin": 3000,
 218                 "tmpdirMin": 4000,
 219                 "outdirMin": 5000
 220             }, {
 221                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 222                 "keep_cache": 512
 223             }, {
 224                 "class": "http://arvados.org/cwl#APIRequirement",
 225             }, {
 226                 "class": "http://arvados.org/cwl#PartitionRequirement",
 227                 "partition": "blurb"
 228             }, {
 229                 "class": "http://arvados.org/cwl#IntermediateOutput",
 230                 "outputTTL": 7200
 231             }, {
 232                 "class": "WorkReuse",
 233                 "enableReuse": False
 234             }],
 235             "baseCommand": "ls",
 236             "id": "",
 237             "class": "CommandLineTool",
 238             "cwlVersion": "v1.2"
 239         })
 240
 241         loadingContext, runtimeContext = self.helper(runner)
 242         runtimeContext.name = "test_resource_requirements"
 243
 244         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 245         arvtool.formatgraph = None
 246         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 247             j.run(runtimeContext)
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         call_body_expected = {
 252             'environment': {
 253                 'HOME': '/var/spool/cwl',
 254                 'TMPDIR': '/tmp'
 255             },
 256             'name': 'test_resource_requirements',
 257             'runtime_constraints': {
 258                 'vcpus': 3,
 259                 'ram': 3145728000,
 260                 'keep_cache_ram': 536870912,
 261                 'API': True
 262             },
 263             'use_existing': False,
 264             'priority': 500,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 4194304000 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 5242880000 }
 270             },
 271             'state': 'Committed',
 272             'output_name': 'Output from step test_resource_requirements',
 273             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 274             'output_path': '/var/spool/cwl',
 275             'output_ttl': 7200,
 276             'container_image': '99999999999999999999999999999993+99',
 277             'command': ['ls'],
 278             'cwd': '/var/spool/cwl',
 279             'scheduling_parameters': {
 280                 'partitions': ['blurb']
 281             },
 282             'properties': {'cwl_input': {}},
 283             'secret_mounts': {},
 284             'output_storage_classes': ["default"]
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # The test passes some fields in builder.resources
 294     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 296     @mock.patch("arvados.collection.Collection")
 297     def test_initial_work_dir(self, collection_mock, keepdocker):
 298         runner = mock.MagicMock()
 299         runner.ignore_docker_for_reuse = False
 300         runner.intermediate_output_ttl = 0
 301         runner.secret_store = cwltool.secrets.SecretStore()
 302         runner.api._rootDesc = {"revision": "20210628"}
 303         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 304
 305         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 306         runner.api.collections().get().execute.return_value = {
 307             "portable_data_hash": "99999999999999999999999999999993+99"}
 308
 309         sourcemock = mock.MagicMock()
 310         def get_collection_mock(p):
 311             if "/" in p:
 312                 return (sourcemock, p.split("/", 1)[1])
 313             else:
 314                 return (sourcemock, "")
 315         runner.fs_access.get_collection.side_effect = get_collection_mock
 316
 317         vwdmock = mock.MagicMock()
 318         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 319
 320         tool = cmap({
 321             "inputs": [],
 322             "outputs": [],
 323             "hints": [{
 324                 "class": "InitialWorkDirRequirement",
 325                 "listing": [{
 326                     "class": "File",
 327                     "basename": "foo",
 328                     "location": "keep:99999999999999999999999999999995+99/bar"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "foo2",
 333                     "location": "keep:99999999999999999999999999999995+99"
 334                 },
 335                 {
 336                     "class": "File",
 337                     "basename": "filename",
 338                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 339                 },
 340                 {
 341                     "class": "Directory",
 342                     "basename": "subdir",
 343                     "location": "keep:99999999999999999999999999999995+99/subdir"
 344                 }                        ]
 345             }],
 346             "baseCommand": "ls",
 347             "class": "CommandLineTool",
 348             "cwlVersion": "v1.2",
 349             "id": ""
 350         })
 351
 352         loadingContext, runtimeContext = self.helper(runner)
 353         runtimeContext.name = "test_initial_work_dir"
 354
 355         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 356
 357         arvtool.formatgraph = None
 358         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 359             j.run(runtimeContext)
 360
 361         call_args, call_kwargs = runner.api.container_requests().create.call_args
 362
 363         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 367
 368         call_body_expected = {
 369             'environment': {
 370                 'HOME': '/var/spool/cwl',
 371                 'TMPDIR': '/tmp'
 372             },
 373             'name': 'test_initial_work_dir',
 374             'runtime_constraints': {
 375                 'vcpus': 1,
 376                 'ram': 268435456
 377             },
 378             'use_existing': True,
 379             'priority': 500,
 380             'mounts': {
 381                 '/tmp': {'kind': 'tmp',
 382                          "capacity": 1073741824 },
 383                 '/var/spool/cwl': {'kind': 'tmp',
 384                                    "capacity": 1073741824 },
 385                 '/var/spool/cwl/foo': {
 386                     'kind': 'collection',
 387                     'path': 'foo',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/foo2': {
 391                     'kind': 'collection',
 392                     'path': 'foo2',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 },
 395                 '/var/spool/cwl/filename': {
 396                     'kind': 'collection',
 397                     'path': 'filename',
 398                     'portable_data_hash': '99999999999999999999999999999996+99'
 399                 },
 400                 '/var/spool/cwl/subdir': {
 401                     'kind': 'collection',
 402                     'path': 'subdir',
 403                     'portable_data_hash': '99999999999999999999999999999996+99'
 404                 }
 405             },
 406             'state': 'Committed',
 407             'output_name': 'Output from step test_initial_work_dir',
 408             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409             'output_path': '/var/spool/cwl',
 410             'output_ttl': 0,
 411             'container_image': '99999999999999999999999999999993+99',
 412             'command': ['ls'],
 413             'cwd': '/var/spool/cwl',
 414             'scheduling_parameters': {
 415             },
 416             'properties': {'cwl_input': {}},
 417             'secret_mounts': {},
 418             'output_storage_classes': ["default"]
 419         }
 420
 421         call_body = call_kwargs.get('body', None)
 422         self.assertNotEqual(None, call_body)
 423         for key in call_body:
 424             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 425
 426
 427     # Test redirecting stdin/stdout/stderr
 428     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 429     def test_redirects(self, keepdocker):
 430         runner = mock.MagicMock()
 431         runner.ignore_docker_for_reuse = False
 432         runner.intermediate_output_ttl = 0
 433         runner.secret_store = cwltool.secrets.SecretStore()
 434         runner.api._rootDesc = {"revision": "20210628"}
 435         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 436
 437         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 438         runner.api.collections().get().execute.return_value = {
 439             "portable_data_hash": "99999999999999999999999999999993+99"}
 440
 441         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 442
 443         tool = cmap({
 444             "inputs": [],
 445             "outputs": [],
 446             "baseCommand": "ls",
 447             "stdout": "stdout.txt",
 448             "stderr": "stderr.txt",
 449             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 450             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 451             "id": "",
 452             "class": "CommandLineTool",
 453             "cwlVersion": "v1.2"
 454         })
 455
 456         loadingContext, runtimeContext = self.helper(runner)
 457         runtimeContext.name = "test_run_redirect"
 458
 459         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 460         arvtool.formatgraph = None
 461         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 462             j.run(runtimeContext)
 463             runner.api.container_requests().create.assert_called_with(
 464                 body=JsonDiffMatcher({
 465                     'environment': {
 466                         'HOME': '/var/spool/cwl',
 467                         'TMPDIR': '/tmp'
 468                     },
 469                     'name': 'test_run_redirect',
 470                     'runtime_constraints': {
 471                         'vcpus': 1,
 472                         'ram': 268435456
 473                     },
 474                     'use_existing': True,
 475                     'priority': 500,
 476                     'mounts': {
 477                         '/tmp': {'kind': 'tmp',
 478                                  "capacity": 1073741824 },
 479                         '/var/spool/cwl': {'kind': 'tmp',
 480                                            "capacity": 1073741824 },
 481                         "stderr": {
 482                             "kind": "file",
 483                             "path": "/var/spool/cwl/stderr.txt"
 484                         },
 485                         "stdin": {
 486                             "kind": "collection",
 487                             "path": "file.txt",
 488                             "portable_data_hash": "99999999999999999999999999999996+99"
 489                         },
 490                         "stdout": {
 491                             "kind": "file",
 492                             "path": "/var/spool/cwl/stdout.txt"
 493                         },
 494                     },
 495                     'state': 'Committed',
 496                     "output_name": "Output from step test_run_redirect",
 497                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 498                     'output_path': '/var/spool/cwl',
 499                     'output_ttl': 0,
 500                     'container_image': '99999999999999999999999999999993+99',
 501                     'command': ['ls', '/var/spool/cwl'],
 502                     'cwd': '/var/spool/cwl',
 503                     'scheduling_parameters': {},
 504                     'properties': {'cwl_input': {}},
 505                     'secret_mounts': {},
 506                     'output_storage_classes': ["default"]
 507                 }))
 508
 509     @mock.patch("arvados.collection.Collection")
 510     def test_done(self, col):
 511         api = mock.MagicMock()
 512
 513         runner = mock.MagicMock()
 514         runner.api = api
 515         runner.num_retries = 0
 516         runner.ignore_docker_for_reuse = False
 517         runner.intermediate_output_ttl = 0
 518         runner.secret_store = cwltool.secrets.SecretStore()
 519
 520         runner.api.containers().get().execute.return_value = {"state":"Complete",
 521                                                               "output": "abc+123",
 522                                                               "exit_code": 0}
 523
 524         col().open.return_value = []
 525
 526         loadingContext, runtimeContext = self.helper(runner)
 527
 528         arvjob = arvados_cwl.ArvadosContainer(runner,
 529                                               runtimeContext,
 530                                               mock.MagicMock(),
 531                                               {},
 532                                               None,
 533                                               [],
 534                                               [],
 535                                               "testjob")
 536         arvjob.output_callback = mock.MagicMock()
 537         arvjob.collect_outputs = mock.MagicMock()
 538         arvjob.successCodes = [0]
 539         arvjob.outdir = "/var/spool/cwl"
 540         arvjob.output_ttl = 3600
 541         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 542
 543         arvjob.collect_outputs.return_value = {"out": "stuff"}
 544
 545         arvjob.done({
 546             "state": "Final",
 547             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 548             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 549             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 550             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 551             "modified_at": "2017-05-26T12:01:22Z",
 552             "properties": {}
 553         })
 554
 555         self.assertFalse(api.collections().create.called)
 556         self.assertFalse(runner.runtime_status_error.called)
 557
 558         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 559         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 560         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 561
 562         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 563                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 564
 565
 566     # Test to make sure we dont call runtime_status_update if we already did
 567     # some where higher up in the call stack
 568     @mock.patch("arvados_cwl.util.get_current_container")
 569     def test_recursive_runtime_status_update(self, gcc_mock):
 570         self.setup_and_test_container_executor_and_logging(gcc_mock)
 571         root_logger = logging.getLogger('')
 572
 573         # get_current_container is invoked when we call runtime_status_update
 574         # so try and log again!
 575         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 576         try:
 577             root_logger.error("First Error")
 578         except RuntimeError:
 579             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 580
 581
 582     # Test to make sure that an exception raised from
 583     # get_current_container doesn't cause the logger to raise an
 584     # exception
 585     @mock.patch("arvados_cwl.util.get_current_container")
 586     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 587         self.setup_and_test_container_executor_and_logging(gcc_mock)
 588         root_logger = logging.getLogger('')
 589
 590         # get_current_container is invoked when we call
 591         # runtime_status_update, it is going to also raise an
 592         # exception.
 593         gcc_mock.side_effect = Exception("Second Error")
 594         try:
 595             root_logger.error("First Error")
 596         except Exception:
 597             self.fail("Exception in logger should not propagate")
 598         self.assertTrue(gcc_mock.called)
 599
 600     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 601     @mock.patch("arvados_cwl.util.get_current_container")
 602     @mock.patch("arvados.collection.CollectionReader")
 603     @mock.patch("arvados.collection.Collection")
 604     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 605         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 606
 607         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 608         self.assertTrue(gcc_mock.called)
 609
 610         runner.num_retries = 0
 611         runner.ignore_docker_for_reuse = False
 612         runner.intermediate_output_ttl = 0
 613         runner.secret_store = cwltool.secrets.SecretStore()
 614         runner.label = mock.MagicMock()
 615         runner.label.return_value = '[container testjob]'
 616
 617         runner.api.containers().get().execute.return_value = {
 618             "state":"Complete",
 619             "output": "abc+123",
 620             "exit_code": 1,
 621             "log": "def+234"
 622         }
 623
 624         col().open.return_value = []
 625
 626         loadingContext, runtimeContext = self.helper(runner)
 627
 628         arvjob = arvados_cwl.ArvadosContainer(runner,
 629                                               runtimeContext,
 630                                               mock.MagicMock(),
 631                                               {},
 632                                               None,
 633                                               [],
 634                                               [],
 635                                               "testjob")
 636         arvjob.output_callback = mock.MagicMock()
 637         arvjob.collect_outputs = mock.MagicMock()
 638         arvjob.successCodes = [0]
 639         arvjob.outdir = "/var/spool/cwl"
 640         arvjob.output_ttl = 3600
 641         arvjob.collect_outputs.return_value = {"out": "stuff"}
 642
 643         arvjob.done({
 644             "state": "Final",
 645             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 646             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 647             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 648             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 649             "modified_at": "2017-05-26T12:01:22Z",
 650             "properties": {}
 651         })
 652
 653         rts_mock.assert_called_with(
 654             'error',
 655             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 656             '  ** log is empty **'
 657         )
 658         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 659
 660     # The test passes no builder.resources
 661     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 662     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 663     def test_mounts(self, keepdocker):
 664         runner = mock.MagicMock()
 665         runner.ignore_docker_for_reuse = False
 666         runner.intermediate_output_ttl = 0
 667         runner.secret_store = cwltool.secrets.SecretStore()
 668         runner.api._rootDesc = {"revision": "20210628"}
 669         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 670
 671         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 672         runner.api.collections().get().execute.return_value = {
 673             "portable_data_hash": "99999999999999999999999999999994+99",
 674             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 675
 676         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 677
 678         tool = cmap({
 679             "inputs": [
 680                 {"id": "p1",
 681                  "type": "Directory"}
 682             ],
 683             "outputs": [],
 684             "baseCommand": "ls",
 685             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 686             "id": "",
 687             "class": "CommandLineTool",
 688             "cwlVersion": "v1.2"
 689         })
 690
 691         loadingContext, runtimeContext = self.helper(runner)
 692         runtimeContext.name = "test_run_mounts"
 693
 694         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 695         arvtool.formatgraph = None
 696         job_order = {
 697             "p1": {
 698                 "class": "Directory",
 699                 "location": "keep:99999999999999999999999999999994+44",
 700                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 701                 "listing": [
 702                     {
 703                         "class": "File",
 704                         "location": "keep:99999999999999999999999999999994+44/file1",
 705                     },
 706                     {
 707                         "class": "File",
 708                         "location": "keep:99999999999999999999999999999994+44/file2",
 709                     }
 710                 ]
 711             }
 712         }
 713         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 714             j.run(runtimeContext)
 715             runner.api.container_requests().create.assert_called_with(
 716                 body=JsonDiffMatcher({
 717                     'environment': {
 718                         'HOME': '/var/spool/cwl',
 719                         'TMPDIR': '/tmp'
 720                     },
 721                     'name': 'test_run_mounts',
 722                     'runtime_constraints': {
 723                         'vcpus': 1,
 724                         'ram': 268435456
 725                     },
 726                     'use_existing': True,
 727                     'priority': 500,
 728                     'mounts': {
 729                         "/keep/99999999999999999999999999999994+44": {
 730                             "kind": "collection",
 731                             "portable_data_hash": "99999999999999999999999999999994+44",
 732                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 733                         },
 734                         '/tmp': {'kind': 'tmp',
 735                                  "capacity": 1073741824 },
 736                         '/var/spool/cwl': {'kind': 'tmp',
 737                                            "capacity": 1073741824 }
 738                     },
 739                     'state': 'Committed',
 740                     'output_name': 'Output from step test_run_mounts',
 741                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 742                     'output_path': '/var/spool/cwl',
 743                     'output_ttl': 0,
 744                     'container_image': '99999999999999999999999999999994+99',
 745                     'command': ['ls', '/var/spool/cwl'],
 746                     'cwd': '/var/spool/cwl',
 747                     'scheduling_parameters': {},
 748                     'properties': {'cwl_input': {
 749                         "p1": {
 750                             "basename": "99999999999999999999999999999994+44",
 751                             "class": "Directory",
 752                             "dirname": "/keep",
 753                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 754                             "listing": [
 755                                 {
 756                                     "basename": "file1",
 757                                     "class": "File",
 758                                     "dirname": "/keep/99999999999999999999999999999994+44",
 759                                     "location": "keep:99999999999999999999999999999994+44/file1",
 760                                     "nameext": "",
 761                                     "nameroot": "file1",
 762                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 763                                     "size": 0
 764                                 },
 765                                 {
 766                                     "basename": "file2",
 767                                     "class": "File",
 768                                     "dirname": "/keep/99999999999999999999999999999994+44",
 769                                     "location": "keep:99999999999999999999999999999994+44/file2",
 770                                     "nameext": "",
 771                                     "nameroot": "file2",
 772                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 773                                     "size": 0
 774                                 }
 775                             ],
 776                             "location": "keep:99999999999999999999999999999994+44",
 777                             "path": "/keep/99999999999999999999999999999994+44"
 778                         }
 779                     }},
 780                     'secret_mounts': {},
 781                     'output_storage_classes': ["default"]
 782                 }))
 783
 784     # The test passes no builder.resources
 785     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 786     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 787     def test_secrets(self, keepdocker):
 788         arvados_cwl.add_arv_hints()
 789         runner = mock.MagicMock()
 790         runner.ignore_docker_for_reuse = False
 791         runner.intermediate_output_ttl = 0
 792         runner.secret_store = cwltool.secrets.SecretStore()
 793         runner.api._rootDesc = {"revision": "20210628"}
 794         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 795
 796         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 797         runner.api.collections().get().execute.return_value = {
 798             "portable_data_hash": "99999999999999999999999999999993+99"}
 799
 800         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 801
 802         tool = cmap({"arguments": ["md5sum", "example.conf"],
 803                      "class": "CommandLineTool",
 804                      "cwlVersion": "v1.2",
 805                      "hints": [
 806                          {
 807                              "class": "http://commonwl.org/cwltool#Secrets",
 808                              "secrets": [
 809                                  "#secret_job.cwl/pw"
 810                              ]
 811                          }
 812                      ],
 813                      "id": "",
 814                      "inputs": [
 815                          {
 816                              "id": "#secret_job.cwl/pw",
 817                              "type": "string"
 818                          }
 819                      ],
 820                      "outputs": [
 821                      ],
 822                      "requirements": [
 823                          {
 824                              "class": "InitialWorkDirRequirement",
 825                              "listing": [
 826                                  {
 827                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 828                                      "entryname": "example.conf"
 829                                  }
 830                              ]
 831                          }
 832                      ]})
 833
 834         loadingContext, runtimeContext = self.helper(runner)
 835         runtimeContext.name = "test_secrets"
 836
 837         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 838         arvtool.formatgraph = None
 839
 840         job_order = {"pw": "blorp"}
 841         runner.secret_store.store(["pw"], job_order)
 842
 843         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 844             j.run(runtimeContext)
 845             runner.api.container_requests().create.assert_called_with(
 846                 body=JsonDiffMatcher({
 847                     'environment': {
 848                         'HOME': '/var/spool/cwl',
 849                         'TMPDIR': '/tmp'
 850                     },
 851                     'name': 'test_secrets',
 852                     'runtime_constraints': {
 853                         'vcpus': 1,
 854                         'ram': 268435456
 855                     },
 856                     'use_existing': True,
 857                     'priority': 500,
 858                     'mounts': {
 859                         '/tmp': {'kind': 'tmp',
 860                                  "capacity": 1073741824
 861                              },
 862                         '/var/spool/cwl': {'kind': 'tmp',
 863                                            "capacity": 1073741824 }
 864                     },
 865                     'state': 'Committed',
 866                     'output_name': 'Output from step test_secrets',
 867                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 868                     'output_path': '/var/spool/cwl',
 869                     'output_ttl': 0,
 870                     'container_image': '99999999999999999999999999999993+99',
 871                     'command': ['md5sum', 'example.conf'],
 872                     'cwd': '/var/spool/cwl',
 873                     'scheduling_parameters': {},
 874                     'properties': {'cwl_input': job_order},
 875                     "secret_mounts": {
 876                         "/var/spool/cwl/example.conf": {
 877                             "content": "username: user\npassword: blorp\n",
 878                             "kind": "text"
 879                         }
 880                     },
 881                     'output_storage_classes': ["default"]
 882                 }))
 883
 884     # The test passes no builder.resources
 885     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 886     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 887     def test_timelimit(self, keepdocker):
 888         runner = mock.MagicMock()
 889         runner.ignore_docker_for_reuse = False
 890         runner.intermediate_output_ttl = 0
 891         runner.secret_store = cwltool.secrets.SecretStore()
 892         runner.api._rootDesc = {"revision": "20210628"}
 893         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 894
 895         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 896         runner.api.collections().get().execute.return_value = {
 897             "portable_data_hash": "99999999999999999999999999999993+99"}
 898
 899         tool = cmap({
 900             "inputs": [],
 901             "outputs": [],
 902             "baseCommand": "ls",
 903             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 904             "id": "",
 905             "cwlVersion": "v1.2",
 906             "class": "CommandLineTool",
 907             "hints": [
 908                 {
 909                     "class": "ToolTimeLimit",
 910                     "timelimit": 42
 911                 }
 912             ]
 913         })
 914
 915         loadingContext, runtimeContext = self.helper(runner)
 916         runtimeContext.name = "test_timelimit"
 917
 918         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 919         arvtool.formatgraph = None
 920
 921         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 922             j.run(runtimeContext)
 923
 924         _, kwargs = runner.api.container_requests().create.call_args
 925         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 926
 927
 928     # The test passes no builder.resources
 929     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 930     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 931     def test_setting_storage_class(self, keepdocker):
 932         arv_docker_clear_cache()
 933
 934         runner = mock.MagicMock()
 935         runner.ignore_docker_for_reuse = False
 936         runner.intermediate_output_ttl = 0
 937         runner.secret_store = cwltool.secrets.SecretStore()
 938         runner.api._rootDesc = {"revision": "20210628"}
 939         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 940
 941         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 942         runner.api.collections().get().execute.return_value = {
 943             "portable_data_hash": "99999999999999999999999999999993+99"}
 944
 945         tool = cmap({
 946             "inputs": [],
 947             "outputs": [],
 948             "baseCommand": "ls",
 949             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 950             "id": "",
 951             "cwlVersion": "v1.2",
 952             "class": "CommandLineTool",
 953             "hints": [
 954                 {
 955                     "class": "http://arvados.org/cwl#OutputStorageClass",
 956                     "finalStorageClass": ["baz_sc", "qux_sc"],
 957                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 958                 }
 959             ]
 960         })
 961
 962         loadingContext, runtimeContext = self.helper(runner, True)
 963
 964         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 965         arvtool.formatgraph = None
 966
 967         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 968             j.run(runtimeContext)
 969             runner.api.container_requests().create.assert_called_with(
 970                 body=JsonDiffMatcher({
 971                     'environment': {
 972                         'HOME': '/var/spool/cwl',
 973                         'TMPDIR': '/tmp'
 974                     },
 975                     'name': 'test_run_True',
 976                     'runtime_constraints': {
 977                         'vcpus': 1,
 978                         'ram': 268435456
 979                     },
 980                     'use_existing': True,
 981                     'priority': 500,
 982                     'mounts': {
 983                         '/tmp': {'kind': 'tmp',
 984                                  "capacity": 1073741824
 985                              },
 986                         '/var/spool/cwl': {'kind': 'tmp',
 987                                            "capacity": 1073741824 }
 988                     },
 989                     'state': 'Committed',
 990                     'output_name': 'Output from step test_run_True',
 991                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 992                     'output_path': '/var/spool/cwl',
 993                     'output_ttl': 0,
 994                     'container_image': '99999999999999999999999999999993+99',
 995                     'command': ['ls', '/var/spool/cwl'],
 996                     'cwd': '/var/spool/cwl',
 997                     'scheduling_parameters': {},
 998                     'properties': {'cwl_input': {}},
 999                     'secret_mounts': {},
1000                     'output_storage_classes': ["foo_sc", "bar_sc"]
1001                 }))
1002
1003
1004     # The test passes no builder.resources
1005     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1006     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1007     def test_setting_process_properties(self, keepdocker):
1008         arv_docker_clear_cache()
1009
1010         runner = mock.MagicMock()
1011         runner.ignore_docker_for_reuse = False
1012         runner.intermediate_output_ttl = 0
1013         runner.secret_store = cwltool.secrets.SecretStore()
1014         runner.api._rootDesc = {"revision": "20210628"}
1015         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1016
1017         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1018         runner.api.collections().get().execute.return_value = {
1019             "portable_data_hash": "99999999999999999999999999999993+99"}
1020
1021         tool = cmap({
1022             "inputs": [
1023                 {"id": "x", "type": "string"}],
1024             "outputs": [],
1025             "baseCommand": "ls",
1026             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1027             "id": "",
1028             "class": "CommandLineTool",
1029             "cwlVersion": "v1.2",
1030             "hints": [
1031             {
1032                 "class": "http://arvados.org/cwl#ProcessProperties",
1033                 "processProperties": [
1034                     {"propertyName": "foo",
1035                      "propertyValue": "bar"},
1036                     {"propertyName": "baz",
1037                      "propertyValue": "$(inputs.x)"},
1038                     {"propertyName": "quux",
1039                      "propertyValue": {
1040                          "q1": 1,
1041                          "q2": 2
1042                      }
1043                     }
1044                 ],
1045             }
1046         ]
1047         })
1048
1049         loadingContext, runtimeContext = self.helper(runner, True)
1050
1051         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1052         arvtool.formatgraph = None
1053
1054         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1055             j.run(runtimeContext)
1056             runner.api.container_requests().create.assert_called_with(
1057                 body=JsonDiffMatcher({
1058                     'environment': {
1059                         'HOME': '/var/spool/cwl',
1060                         'TMPDIR': '/tmp'
1061                     },
1062                     'name': 'test_run_True',
1063                     'runtime_constraints': {
1064                         'vcpus': 1,
1065                         'ram': 268435456
1066                     },
1067                     'use_existing': True,
1068                     'priority': 500,
1069                     'mounts': {
1070                         '/tmp': {'kind': 'tmp',
1071                                  "capacity": 1073741824
1072                              },
1073                         '/var/spool/cwl': {'kind': 'tmp',
1074                                            "capacity": 1073741824 }
1075                     },
1076                     'state': 'Committed',
1077                     'output_name': 'Output from step test_run_True',
1078                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1079                     'output_path': '/var/spool/cwl',
1080                     'output_ttl': 0,
1081                     'container_image': '99999999999999999999999999999993+99',
1082                     'command': ['ls', '/var/spool/cwl'],
1083                     'cwd': '/var/spool/cwl',
1084                     'scheduling_parameters': {},
1085                     'properties': {
1086                         "baz": "blorp",
1087                         "cwl_input": {"x": "blorp"},
1088                         "foo": "bar",
1089                         "quux": {
1090                             "q1": 1,
1091                             "q2": 2
1092                         }
1093                     },
1094                     'secret_mounts': {},
1095                     'output_storage_classes': ["default"]
1096                 }))
1097
1098
1099     # The test passes no builder.resources
1100     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1101     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1102     def test_cuda_requirement(self, keepdocker):
1103         arvados_cwl.add_arv_hints()
1104         arv_docker_clear_cache()
1105
1106         runner = mock.MagicMock()
1107         runner.ignore_docker_for_reuse = False
1108         runner.intermediate_output_ttl = 0
1109         runner.secret_store = cwltool.secrets.SecretStore()
1110         runner.api._rootDesc = {"revision": "20210628"}
1111         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1112
1113         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1114         runner.api.collections().get().execute.return_value = {
1115             "portable_data_hash": "99999999999999999999999999999993+99"}
1116
1117         test_cwl_req = [{
1118                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1119                 "cudaVersionMin": "11.0",
1120                 "cudaComputeCapability": "9.0",
1121             }, {
1122                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1123                 "cudaVersionMin": "11.0",
1124                 "cudaComputeCapability": "9.0",
1125                 "cudaDeviceCountMin": 2
1126             }, {
1127                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1128                 "cudaVersionMin": "11.0",
1129                 "cudaComputeCapability": ["4.0", "5.0"],
1130                 "cudaDeviceCountMin": 2
1131             }]
1132
1133         test_arv_req = [{
1134             'device_count': 1,
1135             'driver_version': "11.0",
1136             'hardware_capability': "9.0"
1137         }, {
1138             'device_count': 2,
1139             'driver_version': "11.0",
1140             'hardware_capability': "9.0"
1141         }, {
1142             'device_count': 2,
1143             'driver_version': "11.0",
1144             'hardware_capability': "4.0"
1145         }]
1146
1147         for test_case in range(0, len(test_cwl_req)):
1148
1149             tool = cmap({
1150                 "inputs": [],
1151                 "outputs": [],
1152                 "baseCommand": "nvidia-smi",
1153                 "arguments": [],
1154                 "id": "",
1155                 "cwlVersion": "v1.2",
1156                 "class": "CommandLineTool",
1157                 "requirements": [test_cwl_req[test_case]]
1158             })
1159
1160             loadingContext, runtimeContext = self.helper(runner, True)
1161
1162             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1163             arvtool.formatgraph = None
1164
1165             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1166                 j.run(runtimeContext)
1167                 runner.api.container_requests().create.assert_called_with(
1168                     body=JsonDiffMatcher({
1169                         'environment': {
1170                             'HOME': '/var/spool/cwl',
1171                             'TMPDIR': '/tmp'
1172                         },
1173                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1174                         'runtime_constraints': {
1175                             'vcpus': 1,
1176                             'ram': 268435456,
1177                             'cuda': test_arv_req[test_case]
1178                         },
1179                         'use_existing': True,
1180                         'priority': 500,
1181                         'mounts': {
1182                             '/tmp': {'kind': 'tmp',
1183                                      "capacity": 1073741824
1184                                  },
1185                             '/var/spool/cwl': {'kind': 'tmp',
1186                                                "capacity": 1073741824 }
1187                         },
1188                         'state': 'Committed',
1189                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1190                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1191                         'output_path': '/var/spool/cwl',
1192                         'output_ttl': 0,
1193                         'container_image': '99999999999999999999999999999993+99',
1194                         'command': ['nvidia-smi'],
1195                         'cwd': '/var/spool/cwl',
1196                         'scheduling_parameters': {},
1197                         'properties': {'cwl_input': {}},
1198                         'secret_mounts': {},
1199                         'output_storage_classes': ["default"]
1200                     }))
1201
1202
1203     # The test passes no builder.resources
1204     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1205     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1206     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1207     def test_match_local_docker(self, keepdocker, determine_image_id):
1208         arvados_cwl.add_arv_hints()
1209         arv_docker_clear_cache()
1210
1211         runner = mock.MagicMock()
1212         runner.ignore_docker_for_reuse = False
1213         runner.intermediate_output_ttl = 0
1214         runner.secret_store = cwltool.secrets.SecretStore()
1215         runner.api._rootDesc = {"revision": "20210628"}
1216         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1217
1218         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1219                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1220         determine_image_id.side_effect = lambda x: "123"
1221         def execute(uuid):
1222             ex = mock.MagicMock()
1223             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1224                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1225             ex.execute.return_value = lookup[uuid]
1226             return ex
1227         runner.api.collections().get.side_effect = execute
1228
1229         tool = cmap({
1230             "inputs": [],
1231             "outputs": [],
1232             "baseCommand": "echo",
1233             "arguments": [],
1234             "id": "",
1235             "cwlVersion": "v1.0",
1236             "class": "org.w3id.cwl.cwl.CommandLineTool"
1237         })
1238
1239         loadingContext, runtimeContext = self.helper(runner, True)
1240
1241         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1242         arvtool.formatgraph = None
1243
1244         container_request = {
1245             'environment': {
1246                 'HOME': '/var/spool/cwl',
1247                 'TMPDIR': '/tmp'
1248             },
1249             'name': 'test_run_True',
1250             'runtime_constraints': {
1251                 'vcpus': 1,
1252                 'ram': 1073741824,
1253             },
1254             'use_existing': True,
1255             'priority': 500,
1256             'mounts': {
1257                 '/tmp': {'kind': 'tmp',
1258                          "capacity": 1073741824
1259                          },
1260                 '/var/spool/cwl': {'kind': 'tmp',
1261                                    "capacity": 1073741824 }
1262             },
1263             'state': 'Committed',
1264             'output_name': 'Output from step test_run_True',
1265             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1266             'output_path': '/var/spool/cwl',
1267             'output_ttl': 0,
1268             'container_image': '99999999999999999999999999999994+99',
1269             'command': ['echo'],
1270             'cwd': '/var/spool/cwl',
1271             'scheduling_parameters': {},
1272             'properties': {'cwl_input': {}},
1273             'secret_mounts': {},
1274             'output_storage_classes': ["default"]
1275         }
1276
1277         runtimeContext.match_local_docker = False
1278         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1279             j.run(runtimeContext)
1280             runner.api.container_requests().create.assert_called_with(
1281                 body=JsonDiffMatcher(container_request))
1282
1283         arv_docker_clear_cache()
1284         runtimeContext.match_local_docker = True
1285         container_request['container_image'] = '99999999999999999999999999999993+99'
1286         container_request['name'] = 'test_run_True_2'
1287         container_request['output_name'] = 'Output from step test_run_True_2'
1288         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1289             j.run(runtimeContext)
1290             runner.api.container_requests().create.assert_called_with(
1291                 body=JsonDiffMatcher(container_request))
1292
1293
1294     # The test passes no builder.resources
1295     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1296     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1297     def test_run_preemptible_hint(self, keepdocker):
1298         arvados_cwl.add_arv_hints()
1299         for enable_preemptible in (None, True, False):
1300             for preemptible_hint in (None, True, False):
1301                 arv_docker_clear_cache()
1302
1303                 runner = mock.MagicMock()
1304                 runner.ignore_docker_for_reuse = False
1305                 runner.intermediate_output_ttl = 0
1306                 runner.secret_store = cwltool.secrets.SecretStore()
1307                 runner.api._rootDesc = {"revision": "20210628"}
1308                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1309
1310                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1311                 runner.api.collections().get().execute.return_value = {
1312                     "portable_data_hash": "99999999999999999999999999999993+99"}
1313
1314                 if preemptible_hint is not None:
1315                     hints = [{
1316                         "class": "http://arvados.org/cwl#UsePreemptible",
1317                         "usePreemptible": preemptible_hint
1318                     }]
1319                 else:
1320                     hints = []
1321
1322                 tool = cmap({
1323                     "inputs": [],
1324                     "outputs": [],
1325                     "baseCommand": "ls",
1326                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1327                     "id": "",
1328                     "class": "CommandLineTool",
1329                     "cwlVersion": "v1.2",
1330                     "hints": hints
1331                 })
1332
1333                 loadingContext, runtimeContext = self.helper(runner)
1334
1335                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1336                 runtimeContext.enable_preemptible = enable_preemptible
1337
1338                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1339                 arvtool.formatgraph = None
1340
1341                 # Test the interactions between --enable/disable-preemptible
1342                 # and UsePreemptible hint
1343
1344                 if enable_preemptible is None:
1345                     if preemptible_hint is None:
1346                         sched = {}
1347                     else:
1348                         sched = {'preemptible': preemptible_hint}
1349                 else:
1350                     if preemptible_hint is None:
1351                         sched = {'preemptible': enable_preemptible}
1352                     else:
1353                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1354
1355                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1356                     j.run(runtimeContext)
1357                     runner.api.container_requests().create.assert_called_with(
1358                         body=JsonDiffMatcher({
1359                             'environment': {
1360                                 'HOME': '/var/spool/cwl',
1361                                 'TMPDIR': '/tmp'
1362                             },
1363                             'name': runtimeContext.name,
1364                             'runtime_constraints': {
1365                                 'vcpus': 1,
1366                                 'ram': 268435456
1367                             },
1368                             'use_existing': True,
1369                             'priority': 500,
1370                             'mounts': {
1371                                 '/tmp': {'kind': 'tmp',
1372                                          "capacity": 1073741824
1373                                      },
1374                                 '/var/spool/cwl': {'kind': 'tmp',
1375                                                    "capacity": 1073741824 }
1376                             },
1377                             'state': 'Committed',
1378                             'output_name': 'Output from step '+runtimeContext.name,
1379                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1380                             'output_path': '/var/spool/cwl',
1381                             'output_ttl': 0,
1382                             'container_image': '99999999999999999999999999999993+99',
1383                             'command': ['ls', '/var/spool/cwl'],
1384                             'cwd': '/var/spool/cwl',
1385                             'scheduling_parameters': sched,
1386                             'properties': {'cwl_input': {}},
1387                             'secret_mounts': {},
1388                             'output_storage_classes': ["default"]
1389                         }))
1390
1391
1392     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1393     def test_output_properties(self, keepdocker):
1394         arvados_cwl.add_arv_hints()
1395         for rev in ["20210628", "20220510"]:
1396             runner = mock.MagicMock()
1397             runner.ignore_docker_for_reuse = False
1398             runner.intermediate_output_ttl = 0
1399             runner.secret_store = cwltool.secrets.SecretStore()
1400             runner.api._rootDesc = {"revision": rev}
1401             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1402
1403             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1404             runner.api.collections().get().execute.return_value = {
1405                 "portable_data_hash": "99999999999999999999999999999993+99"}
1406
1407             tool = cmap({
1408                 "inputs": [{
1409                     "id": "inp",
1410                     "type": "string"
1411                 }],
1412                 "outputs": [],
1413                 "baseCommand": "ls",
1414                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1415                 "id": "",
1416                 "cwlVersion": "v1.2",
1417                 "class": "CommandLineTool",
1418                 "hints": [
1419                     {
1420                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1421                         "outputProperties": {
1422                             "foo": "bar",
1423                             "baz": "$(inputs.inp)"
1424                         }
1425                     }
1426                 ]
1427             })
1428
1429             loadingContext, runtimeContext = self.helper(runner)
1430             runtimeContext.name = "test_timelimit"
1431
1432             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1433             arvtool.formatgraph = None
1434
1435             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1436                 j.run(runtimeContext)
1437
1438             _, kwargs = runner.api.container_requests().create.call_args
1439             if rev == "20220510":
1440                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1441             else:
1442                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1443
1444
1445 class TestWorkflow(unittest.TestCase):
1446     def setUp(self):
1447         cwltool.process._names = set()
1448         arv_docker_clear_cache()
1449
1450     def helper(self, runner, enable_reuse=True):
1451         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1452
1453         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1454                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1455
1456         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1457         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1458         document_loader.fetch_text = document_loader.fetcher.fetch_text
1459         document_loader.check_exists = document_loader.fetcher.check_exists
1460
1461         loadingContext = arvados_cwl.context.ArvLoadingContext(
1462             {"avsc_names": avsc_names,
1463              "basedir": "",
1464              "make_fs_access": make_fs_access,
1465              "loader": document_loader,
1466              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1467              "construct_tool_object": runner.arv_make_tool})
1468         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1469             {"work_api": "containers",
1470              "basedir": "",
1471              "name": "test_run_wf_"+str(enable_reuse),
1472              "make_fs_access": make_fs_access,
1473              "tmpdir": "/tmp",
1474              "enable_reuse": enable_reuse,
1475              "priority": 500})
1476
1477         return loadingContext, runtimeContext
1478
1479     # The test passes no builder.resources
1480     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1481     @mock.patch("arvados.collection.CollectionReader")
1482     @mock.patch("arvados.collection.Collection")
1483     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1484     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1485         arvados_cwl.add_arv_hints()
1486
1487         api = mock.MagicMock()
1488         api._rootDesc = get_rootDesc()
1489         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1490
1491         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1492         self.assertEqual(runner.work_api, 'containers')
1493
1494         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1495         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1496         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1497                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1498
1499         runner.api.containers().current().execute.return_value = {}
1500
1501         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1502         runner.ignore_docker_for_reuse = False
1503         runner.num_retries = 0
1504         runner.secret_store = cwltool.secrets.SecretStore()
1505
1506         loadingContext, runtimeContext = self.helper(runner)
1507         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1508
1509         mockcollectionreader().exists.return_value = True
1510
1511         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1512         metadata["cwlVersion"] = tool["cwlVersion"]
1513
1514         mockc = mock.MagicMock()
1515         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1516         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1517
1518         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1519         arvtool.formatgraph = None
1520         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1521
1522         next(it).run(runtimeContext)
1523         next(it).run(runtimeContext)
1524
1525         with open("tests/wf/scatter2_subwf.cwl") as f:
1526             subwf = StripYAMLComments(f.read()).rstrip()
1527
1528         runner.api.container_requests().create.assert_called_with(
1529             body=JsonDiffMatcher({
1530                 "command": [
1531                     "cwltool",
1532                     "--no-container",
1533                     "--move-outputs",
1534                     "--preserve-entire-environment",
1535                     "workflow.cwl",
1536                     "cwl.input.yml"
1537                 ],
1538                 "container_image": "99999999999999999999999999999993+99",
1539                 "cwd": "/var/spool/cwl",
1540                 "environment": {
1541                     "HOME": "/var/spool/cwl",
1542                     "TMPDIR": "/tmp"
1543                 },
1544                 "mounts": {
1545                     "/keep/99999999999999999999999999999999+118": {
1546                         "kind": "collection",
1547                         "portable_data_hash": "99999999999999999999999999999999+118"
1548                     },
1549                     "/tmp": {
1550                         "capacity": 1073741824,
1551                         "kind": "tmp"
1552                     },
1553                     "/var/spool/cwl": {
1554                         "capacity": 1073741824,
1555                         "kind": "tmp"
1556                     },
1557                     "/var/spool/cwl/cwl.input.yml": {
1558                         "kind": "collection",
1559                         "path": "cwl.input.yml",
1560                         "portable_data_hash": "99999999999999999999999999999996+99"
1561                     },
1562                     "/var/spool/cwl/workflow.cwl": {
1563                         "kind": "collection",
1564                         "path": "workflow.cwl",
1565                         "portable_data_hash": "99999999999999999999999999999996+99"
1566                     },
1567                     "stdout": {
1568                         "kind": "file",
1569                         "path": "/var/spool/cwl/cwl.output.json"
1570                     }
1571                 },
1572                 "name": "scatterstep",
1573                 "output_name": "Output from step scatterstep",
1574                 "output_path": "/var/spool/cwl",
1575                 "output_ttl": 0,
1576                 "priority": 500,
1577                 "properties": {'cwl_input': {
1578                         "fileblub": {
1579                             "basename": "token.txt",
1580                             "class": "File",
1581                             "dirname": "/keep/99999999999999999999999999999999+118",
1582                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1583                             "nameext": ".txt",
1584                             "nameroot": "token",
1585                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1586                             "size": 0
1587                         },
1588                         "sleeptime": 5
1589                 }},
1590                 "runtime_constraints": {
1591                     "ram": 1073741824,
1592                     "vcpus": 1
1593                 },
1594                 "scheduling_parameters": {},
1595                 "secret_mounts": {},
1596                 "state": "Committed",
1597                 "use_existing": True,
1598                 'output_storage_classes': ["default"]
1599             }))
1600         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1601         mockc.open().__enter__().write.assert_has_calls([mock.call(
1602 '''{
1603   "fileblub": {
1604     "basename": "token.txt",
1605     "class": "File",
1606     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1607     "size": 0
1608   },
1609   "sleeptime": 5
1610 }''')])
1611
1612     # The test passes no builder.resources
1613     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1614     @mock.patch("arvados.collection.CollectionReader")
1615     @mock.patch("arvados.collection.Collection")
1616     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1617     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1618         arvados_cwl.add_arv_hints()
1619
1620         api = mock.MagicMock()
1621         api._rootDesc = get_rootDesc()
1622         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1623
1624         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1625         self.assertEqual(runner.work_api, 'containers')
1626
1627         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1628         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1629                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1630         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1631                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1632
1633         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1634         runner.ignore_docker_for_reuse = False
1635         runner.num_retries = 0
1636         runner.secret_store = cwltool.secrets.SecretStore()
1637
1638         loadingContext, runtimeContext = self.helper(runner)
1639         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1640         loadingContext.do_update = True
1641         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1642
1643         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1644
1645         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1646         arvtool.formatgraph = None
1647         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1648
1649         next(it).run(runtimeContext)
1650         next(it).run(runtimeContext)
1651
1652         with open("tests/wf/echo-subwf.cwl") as f:
1653             subwf = StripYAMLComments(f.read())
1654
1655         runner.api.container_requests().create.assert_called_with(
1656             body=JsonDiffMatcher({
1657                 'output_ttl': 0,
1658                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1659                 'scheduling_parameters': {},
1660                 'name': u'echo-subwf',
1661                 'secret_mounts': {},
1662                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1663                 'properties': {'cwl_input': {}},
1664                 'priority': 500,
1665                 'mounts': {
1666                     '/var/spool/cwl/cwl.input.yml': {
1667                         'portable_data_hash': '99999999999999999999999999999996+99',
1668                         'kind': 'collection',
1669                         'path': 'cwl.input.yml'
1670                     },
1671                     '/var/spool/cwl/workflow.cwl': {
1672                         'portable_data_hash': '99999999999999999999999999999996+99',
1673                         'kind': 'collection',
1674                         'path': 'workflow.cwl'
1675                     },
1676                     'stdout': {
1677                         'path': '/var/spool/cwl/cwl.output.json',
1678                         'kind': 'file'
1679                     },
1680                     '/tmp': {
1681                         'kind': 'tmp',
1682                         'capacity': 1073741824
1683                     }, '/var/spool/cwl': {
1684                         'kind': 'tmp',
1685                         'capacity': 3221225472
1686                     }
1687                 },
1688                 'state': 'Committed',
1689                 'output_path': '/var/spool/cwl',
1690                 'container_image': '99999999999999999999999999999993+99',
1691                 'command': [
1692                     u'cwltool',
1693                     u'--no-container',
1694                     u'--move-outputs',
1695                     u'--preserve-entire-environment',
1696                     u'workflow.cwl',
1697                     u'cwl.input.yml'
1698                 ],
1699                 'use_existing': True,
1700                 'output_name': u'Output from step echo-subwf',
1701                 'cwd': '/var/spool/cwl',
1702                 'output_storage_classes': ["default"]
1703             }))
1704
1705     def test_default_work_api(self):
1706         arvados_cwl.add_arv_hints()
1707
1708         api = mock.MagicMock()
1709         api._rootDesc = copy.deepcopy(get_rootDesc())
1710         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1711         self.assertEqual(runner.work_api, 'containers')