sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 #from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         #arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             #arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 139
 140             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 141             runner.api.collections().get().execute.return_value = {
 142                 "portable_data_hash": "99999999999999999999999999999993+99"}
 143
 144             tool = cmap({
 145                 "inputs": [],
 146                 "outputs": [],
 147                 "baseCommand": "ls",
 148                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 149                 "id": "",
 150                 "class": "CommandLineTool",
 151                 "cwlVersion": "v1.2"
 152             })
 153
 154             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 155
 156             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 157             arvtool.formatgraph = None
 158
 159             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 160                 j.run(runtimeContext)
 161                 runner.api.container_requests().create.assert_called_with(
 162                     body=JsonDiffMatcher({
 163                         'environment': {
 164                             'HOME': '/var/spool/cwl',
 165                             'TMPDIR': '/tmp'
 166                         },
 167                         'name': 'test_run_'+str(enable_reuse),
 168                         'runtime_constraints': {
 169                             'vcpus': 1,
 170                             'ram': 268435456
 171                         },
 172                         'use_existing': enable_reuse,
 173                         'priority': 500,
 174                         'mounts': {
 175                             '/tmp': {'kind': 'tmp',
 176                                      "capacity": 1073741824
 177                                  },
 178                             '/var/spool/cwl': {'kind': 'tmp',
 179                                                "capacity": 1073741824 }
 180                         },
 181                         'state': 'Committed',
 182                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 183                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                         'output_path': '/var/spool/cwl',
 185                         'output_ttl': 0,
 186                         'container_image': '99999999999999999999999999999993+99',
 187                         'command': ['ls', '/var/spool/cwl'],
 188                         'cwd': '/var/spool/cwl',
 189                         'scheduling_parameters': {},
 190                         'properties': {'cwl_input': {}},
 191                         'secret_mounts': {},
 192                         'output_storage_classes': ["default"]
 193                     }))
 194
 195     # The test passes some fields in builder.resources
 196     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 198     def test_resource_requirements(self, keepdocker):
 199         arvados_cwl.add_arv_hints()
 200         runner = mock.MagicMock()
 201         runner.ignore_docker_for_reuse = False
 202         runner.intermediate_output_ttl = 3600
 203         runner.secret_store = cwltool.secrets.SecretStore()
 204         runner.api._rootDesc = {"revision": "20210628"}
 205         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 206
 207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 208         runner.api.collections().get().execute.return_value = {
 209             "portable_data_hash": "99999999999999999999999999999993+99"}
 210
 211         tool = cmap({
 212             "inputs": [],
 213             "outputs": [],
 214             "hints": [{
 215                 "class": "ResourceRequirement",
 216                 "coresMin": 3,
 217                 "ramMin": 3000,
 218                 "tmpdirMin": 4000,
 219                 "outdirMin": 5000
 220             }, {
 221                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 222                 "keep_cache": 512
 223             }, {
 224                 "class": "http://arvados.org/cwl#APIRequirement",
 225             }, {
 226                 "class": "http://arvados.org/cwl#PartitionRequirement",
 227                 "partition": "blurb"
 228             }, {
 229                 "class": "http://arvados.org/cwl#IntermediateOutput",
 230                 "outputTTL": 7200
 231             }, {
 232                 "class": "WorkReuse",
 233                 "enableReuse": False
 234             }],
 235             "baseCommand": "ls",
 236             "id": "",
 237             "class": "CommandLineTool",
 238             "cwlVersion": "v1.2"
 239         })
 240
 241         loadingContext, runtimeContext = self.helper(runner)
 242         runtimeContext.name = "test_resource_requirements"
 243
 244         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 245         arvtool.formatgraph = None
 246         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 247             j.run(runtimeContext)
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         call_body_expected = {
 252             'environment': {
 253                 'HOME': '/var/spool/cwl',
 254                 'TMPDIR': '/tmp'
 255             },
 256             'name': 'test_resource_requirements',
 257             'runtime_constraints': {
 258                 'vcpus': 3,
 259                 'ram': 3145728000,
 260                 'keep_cache_ram': 536870912,
 261                 'API': True
 262             },
 263             'use_existing': False,
 264             'priority': 500,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 4194304000 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 5242880000 }
 270             },
 271             'state': 'Committed',
 272             'output_name': 'Output from step test_resource_requirements',
 273             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 274             'output_path': '/var/spool/cwl',
 275             'output_ttl': 7200,
 276             'container_image': '99999999999999999999999999999993+99',
 277             'command': ['ls'],
 278             'cwd': '/var/spool/cwl',
 279             'scheduling_parameters': {
 280                 'partitions': ['blurb']
 281             },
 282             'properties': {'cwl_input': {}},
 283             'secret_mounts': {},
 284             'output_storage_classes': ["default"]
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # The test passes some fields in builder.resources
 294     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 296     @mock.patch("arvados.collection.Collection")
 297     def test_initial_work_dir(self, collection_mock, keepdocker):
 298         runner = mock.MagicMock()
 299         runner.ignore_docker_for_reuse = False
 300         runner.intermediate_output_ttl = 0
 301         runner.secret_store = cwltool.secrets.SecretStore()
 302         runner.api._rootDesc = {"revision": "20210628"}
 303         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 304
 305         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 306         runner.api.collections().get().execute.return_value = {
 307             "portable_data_hash": "99999999999999999999999999999993+99"}
 308
 309         sourcemock = mock.MagicMock()
 310         def get_collection_mock(p):
 311             if "/" in p:
 312                 return (sourcemock, p.split("/", 1)[1])
 313             else:
 314                 return (sourcemock, "")
 315         runner.fs_access.get_collection.side_effect = get_collection_mock
 316
 317         vwdmock = mock.MagicMock()
 318         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 319
 320         tool = cmap({
 321             "inputs": [],
 322             "outputs": [],
 323             "hints": [{
 324                 "class": "InitialWorkDirRequirement",
 325                 "listing": [{
 326                     "class": "File",
 327                     "basename": "foo",
 328                     "location": "keep:99999999999999999999999999999995+99/bar"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "foo2",
 333                     "location": "keep:99999999999999999999999999999995+99"
 334                 },
 335                 {
 336                     "class": "File",
 337                     "basename": "filename",
 338                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 339                 },
 340                 {
 341                     "class": "Directory",
 342                     "basename": "subdir",
 343                     "location": "keep:99999999999999999999999999999995+99/subdir"
 344                 }                        ]
 345             }],
 346             "baseCommand": "ls",
 347             "class": "CommandLineTool",
 348             "cwlVersion": "v1.2",
 349             "id": ""
 350         })
 351
 352         loadingContext, runtimeContext = self.helper(runner)
 353         runtimeContext.name = "test_initial_work_dir"
 354
 355         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 356
 357         arvtool.formatgraph = None
 358         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 359             j.run(runtimeContext)
 360
 361         call_args, call_kwargs = runner.api.container_requests().create.call_args
 362
 363         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 367
 368         call_body_expected = {
 369             'environment': {
 370                 'HOME': '/var/spool/cwl',
 371                 'TMPDIR': '/tmp'
 372             },
 373             'name': 'test_initial_work_dir',
 374             'runtime_constraints': {
 375                 'vcpus': 1,
 376                 'ram': 268435456
 377             },
 378             'use_existing': True,
 379             'priority': 500,
 380             'mounts': {
 381                 '/tmp': {'kind': 'tmp',
 382                          "capacity": 1073741824 },
 383                 '/var/spool/cwl': {'kind': 'tmp',
 384                                    "capacity": 1073741824 },
 385                 '/var/spool/cwl/foo': {
 386                     'kind': 'collection',
 387                     'path': 'foo',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/foo2': {
 391                     'kind': 'collection',
 392                     'path': 'foo2',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 },
 395                 '/var/spool/cwl/filename': {
 396                     'kind': 'collection',
 397                     'path': 'filename',
 398                     'portable_data_hash': '99999999999999999999999999999996+99'
 399                 },
 400                 '/var/spool/cwl/subdir': {
 401                     'kind': 'collection',
 402                     'path': 'subdir',
 403                     'portable_data_hash': '99999999999999999999999999999996+99'
 404                 }
 405             },
 406             'state': 'Committed',
 407             'output_name': 'Output from step test_initial_work_dir',
 408             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409             'output_path': '/var/spool/cwl',
 410             'output_ttl': 0,
 411             'container_image': '99999999999999999999999999999993+99',
 412             'command': ['ls'],
 413             'cwd': '/var/spool/cwl',
 414             'scheduling_parameters': {
 415             },
 416             'properties': {'cwl_input': {}},
 417             'secret_mounts': {},
 418             'output_storage_classes': ["default"]
 419         }
 420
 421         call_body = call_kwargs.get('body', None)
 422         self.assertNotEqual(None, call_body)
 423         for key in call_body:
 424             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 425
 426
 427     # Test redirecting stdin/stdout/stderr
 428     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 429     def test_redirects(self, keepdocker):
 430         runner = mock.MagicMock()
 431         runner.ignore_docker_for_reuse = False
 432         runner.intermediate_output_ttl = 0
 433         runner.secret_store = cwltool.secrets.SecretStore()
 434         runner.api._rootDesc = {"revision": "20210628"}
 435         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 436
 437         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 438         runner.api.collections().get().execute.return_value = {
 439             "portable_data_hash": "99999999999999999999999999999993+99"}
 440
 441         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 442
 443         tool = cmap({
 444             "inputs": [],
 445             "outputs": [],
 446             "baseCommand": "ls",
 447             "stdout": "stdout.txt",
 448             "stderr": "stderr.txt",
 449             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 450             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 451             "id": "",
 452             "class": "CommandLineTool",
 453             "cwlVersion": "v1.2"
 454         })
 455
 456         loadingContext, runtimeContext = self.helper(runner)
 457         runtimeContext.name = "test_run_redirect"
 458
 459         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 460         arvtool.formatgraph = None
 461         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 462             j.run(runtimeContext)
 463             runner.api.container_requests().create.assert_called_with(
 464                 body=JsonDiffMatcher({
 465                     'environment': {
 466                         'HOME': '/var/spool/cwl',
 467                         'TMPDIR': '/tmp'
 468                     },
 469                     'name': 'test_run_redirect',
 470                     'runtime_constraints': {
 471                         'vcpus': 1,
 472                         'ram': 268435456
 473                     },
 474                     'use_existing': True,
 475                     'priority': 500,
 476                     'mounts': {
 477                         '/tmp': {'kind': 'tmp',
 478                                  "capacity": 1073741824 },
 479                         '/var/spool/cwl': {'kind': 'tmp',
 480                                            "capacity": 1073741824 },
 481                         "stderr": {
 482                             "kind": "file",
 483                             "path": "/var/spool/cwl/stderr.txt"
 484                         },
 485                         "stdin": {
 486                             "kind": "collection",
 487                             "path": "file.txt",
 488                             "portable_data_hash": "99999999999999999999999999999996+99"
 489                         },
 490                         "stdout": {
 491                             "kind": "file",
 492                             "path": "/var/spool/cwl/stdout.txt"
 493                         },
 494                     },
 495                     'state': 'Committed',
 496                     "output_name": "Output from step test_run_redirect",
 497                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 498                     'output_path': '/var/spool/cwl',
 499                     'output_ttl': 0,
 500                     'container_image': '99999999999999999999999999999993+99',
 501                     'command': ['ls', '/var/spool/cwl'],
 502                     'cwd': '/var/spool/cwl',
 503                     'scheduling_parameters': {},
 504                     'properties': {'cwl_input': {}},
 505                     'secret_mounts': {},
 506                     'output_storage_classes': ["default"]
 507                 }))
 508
 509     @mock.patch("arvados.collection.Collection")
 510     def test_done(self, col):
 511         api = mock.MagicMock()
 512
 513         runner = mock.MagicMock()
 514         runner.api = api
 515         runner.num_retries = 0
 516         runner.ignore_docker_for_reuse = False
 517         runner.intermediate_output_ttl = 0
 518         runner.secret_store = cwltool.secrets.SecretStore()
 519
 520         runner.api.containers().get().execute.return_value = {"state":"Complete",
 521                                                               "output": "abc+123",
 522                                                               "exit_code": 0}
 523
 524         col().open.return_value = []
 525
 526         loadingContext, runtimeContext = self.helper(runner)
 527
 528         arvjob = arvados_cwl.ArvadosContainer(runner,
 529                                               runtimeContext,
 530                                               mock.MagicMock(),
 531                                               {},
 532                                               None,
 533                                               [],
 534                                               [],
 535                                               "testjob")
 536         arvjob.output_callback = mock.MagicMock()
 537         arvjob.collect_outputs = mock.MagicMock()
 538         arvjob.successCodes = [0]
 539         arvjob.outdir = "/var/spool/cwl"
 540         arvjob.output_ttl = 3600
 541         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 542
 543         arvjob.collect_outputs.return_value = {"out": "stuff"}
 544
 545         arvjob.done({
 546             "state": "Final",
 547             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 548             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 549             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 550             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 551             "modified_at": "2017-05-26T12:01:22Z",
 552             "properties": {}
 553         })
 554
 555         self.assertFalse(api.collections().create.called)
 556         self.assertFalse(runner.runtime_status_error.called)
 557
 558         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 559         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 560         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 561
 562         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 563                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 564
 565
 566     # Test to make sure we dont call runtime_status_update if we already did
 567     # some where higher up in the call stack
 568     @mock.patch("arvados_cwl.util.get_current_container")
 569     def test_recursive_runtime_status_update(self, gcc_mock):
 570         self.setup_and_test_container_executor_and_logging(gcc_mock)
 571         root_logger = logging.getLogger('')
 572
 573         # get_current_container is invoked when we call runtime_status_update
 574         # so try and log again!
 575         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 576         try:
 577             root_logger.error("First Error")
 578         except RuntimeError:
 579             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 580
 581
 582     # Test to make sure that an exception raised from
 583     # get_current_container doesn't cause the logger to raise an
 584     # exception
 585     @mock.patch("arvados_cwl.util.get_current_container")
 586     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 587         self.setup_and_test_container_executor_and_logging(gcc_mock)
 588         root_logger = logging.getLogger('')
 589
 590         # get_current_container is invoked when we call
 591         # runtime_status_update, it is going to also raise an
 592         # exception.
 593         gcc_mock.side_effect = Exception("Second Error")
 594         try:
 595             root_logger.error("First Error")
 596         except Exception:
 597             self.fail("Exception in logger should not propagate")
 598         self.assertTrue(gcc_mock.called)
 599
 600     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 601     @mock.patch("arvados_cwl.util.get_current_container")
 602     @mock.patch("arvados.collection.CollectionReader")
 603     @mock.patch("arvados.collection.Collection")
 604     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 605         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 606
 607         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 608         self.assertTrue(gcc_mock.called)
 609
 610         runner.num_retries = 0
 611         runner.ignore_docker_for_reuse = False
 612         runner.intermediate_output_ttl = 0
 613         runner.secret_store = cwltool.secrets.SecretStore()
 614         runner.label = mock.MagicMock()
 615         runner.label.return_value = '[container testjob]'
 616
 617         runner.api.containers().get().execute.return_value = {
 618             "state":"Complete",
 619             "output": "abc+123",
 620             "exit_code": 1,
 621             "log": "def+234"
 622         }
 623
 624         col().open.return_value = []
 625
 626         loadingContext, runtimeContext = self.helper(runner)
 627
 628         arvjob = arvados_cwl.ArvadosContainer(runner,
 629                                               runtimeContext,
 630                                               mock.MagicMock(),
 631                                               {},
 632                                               None,
 633                                               [],
 634                                               [],
 635                                               "testjob")
 636         arvjob.output_callback = mock.MagicMock()
 637         arvjob.collect_outputs = mock.MagicMock()
 638         arvjob.successCodes = [0]
 639         arvjob.outdir = "/var/spool/cwl"
 640         arvjob.output_ttl = 3600
 641         arvjob.collect_outputs.return_value = {"out": "stuff"}
 642
 643         arvjob.done({
 644             "state": "Final",
 645             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 646             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 647             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 648             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 649             "modified_at": "2017-05-26T12:01:22Z",
 650             "properties": {}
 651         })
 652
 653         rts_mock.assert_called_with(
 654             'error',
 655             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 656             '  ** log is empty **'
 657         )
 658         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 659
 660     # The test passes no builder.resources
 661     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 662     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 663     def test_mounts(self, keepdocker):
 664         runner = mock.MagicMock()
 665         runner.ignore_docker_for_reuse = False
 666         runner.intermediate_output_ttl = 0
 667         runner.secret_store = cwltool.secrets.SecretStore()
 668         runner.api._rootDesc = {"revision": "20210628"}
 669         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 670
 671         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 672         runner.api.collections().get().execute.return_value = {
 673             "portable_data_hash": "99999999999999999999999999999994+99",
 674             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 675
 676         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 677
 678         tool = cmap({
 679             "inputs": [
 680                 {"id": "p1",
 681                  "type": "Directory"}
 682             ],
 683             "outputs": [],
 684             "baseCommand": "ls",
 685             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 686             "id": "",
 687             "class": "CommandLineTool",
 688             "cwlVersion": "v1.2"
 689         })
 690
 691         loadingContext, runtimeContext = self.helper(runner)
 692         runtimeContext.name = "test_run_mounts"
 693
 694         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 695         arvtool.formatgraph = None
 696         job_order = {
 697             "p1": {
 698                 "class": "Directory",
 699                 "location": "keep:99999999999999999999999999999994+44",
 700                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 701                 "listing": [
 702                     {
 703                         "class": "File",
 704                         "location": "keep:99999999999999999999999999999994+44/file1",
 705                     },
 706                     {
 707                         "class": "File",
 708                         "location": "keep:99999999999999999999999999999994+44/file2",
 709                     }
 710                 ]
 711             }
 712         }
 713         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 714             j.run(runtimeContext)
 715             runner.api.container_requests().create.assert_called_with(
 716                 body=JsonDiffMatcher({
 717                     'environment': {
 718                         'HOME': '/var/spool/cwl',
 719                         'TMPDIR': '/tmp'
 720                     },
 721                     'name': 'test_run_mounts',
 722                     'runtime_constraints': {
 723                         'vcpus': 1,
 724                         'ram': 268435456
 725                     },
 726                     'use_existing': True,
 727                     'priority': 500,
 728                     'mounts': {
 729                         "/keep/99999999999999999999999999999994+44": {
 730                             "kind": "collection",
 731                             "portable_data_hash": "99999999999999999999999999999994+44",
 732                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 733                         },
 734                         '/tmp': {'kind': 'tmp',
 735                                  "capacity": 1073741824 },
 736                         '/var/spool/cwl': {'kind': 'tmp',
 737                                            "capacity": 1073741824 }
 738                     },
 739                     'state': 'Committed',
 740                     'output_name': 'Output from step test_run_mounts',
 741                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 742                     'output_path': '/var/spool/cwl',
 743                     'output_ttl': 0,
 744                     'container_image': '99999999999999999999999999999994+99',
 745                     'command': ['ls', '/var/spool/cwl'],
 746                     'cwd': '/var/spool/cwl',
 747                     'scheduling_parameters': {},
 748                     'properties': {'cwl_input': {
 749                         "p1": {
 750                             "basename": "99999999999999999999999999999994+44",
 751                             "class": "Directory",
 752                             "dirname": "/keep",
 753                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 754                             "listing": [
 755                                 {
 756                                     "basename": "file1",
 757                                     "class": "File",
 758                                     "dirname": "/keep/99999999999999999999999999999994+44",
 759                                     "location": "keep:99999999999999999999999999999994+44/file1",
 760                                     "nameext": "",
 761                                     "nameroot": "file1",
 762                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 763                                     "size": 0
 764                                 },
 765                                 {
 766                                     "basename": "file2",
 767                                     "class": "File",
 768                                     "dirname": "/keep/99999999999999999999999999999994+44",
 769                                     "location": "keep:99999999999999999999999999999994+44/file2",
 770                                     "nameext": "",
 771                                     "nameroot": "file2",
 772                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 773                                     "size": 0
 774                                 }
 775                             ],
 776                             "location": "keep:99999999999999999999999999999994+44",
 777                             "path": "/keep/99999999999999999999999999999994+44"
 778                         }
 779                     }},
 780                     'secret_mounts': {},
 781                     'output_storage_classes': ["default"]
 782                 }))
 783
 784     # The test passes no builder.resources
 785     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 786     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 787     def test_secrets(self, keepdocker):
 788         arvados_cwl.add_arv_hints()
 789         runner = mock.MagicMock()
 790         runner.ignore_docker_for_reuse = False
 791         runner.intermediate_output_ttl = 0
 792         runner.secret_store = cwltool.secrets.SecretStore()
 793         runner.api._rootDesc = {"revision": "20210628"}
 794         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 795
 796         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 797         runner.api.collections().get().execute.return_value = {
 798             "portable_data_hash": "99999999999999999999999999999993+99"}
 799
 800         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 801
 802         tool = cmap({"arguments": ["md5sum", "example.conf"],
 803                      "class": "CommandLineTool",
 804                      "cwlVersion": "v1.2",
 805                      "hints": [
 806                          {
 807                              "class": "http://commonwl.org/cwltool#Secrets",
 808                              "secrets": [
 809                                  "#secret_job.cwl/pw"
 810                              ]
 811                          }
 812                      ],
 813                      "id": "",
 814                      "inputs": [
 815                          {
 816                              "id": "#secret_job.cwl/pw",
 817                              "type": "string"
 818                          }
 819                      ],
 820                      "outputs": [
 821                      ],
 822                      "requirements": [
 823                          {
 824                              "class": "InitialWorkDirRequirement",
 825                              "listing": [
 826                                  {
 827                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 828                                      "entryname": "example.conf"
 829                                  }
 830                              ]
 831                          }
 832                      ]})
 833
 834         loadingContext, runtimeContext = self.helper(runner)
 835         runtimeContext.name = "test_secrets"
 836
 837         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 838         arvtool.formatgraph = None
 839
 840         job_order = {"pw": "blorp"}
 841         runner.secret_store.store(["pw"], job_order)
 842
 843         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 844             j.run(runtimeContext)
 845             runner.api.container_requests().create.assert_called_with(
 846                 body=JsonDiffMatcher({
 847                     'environment': {
 848                         'HOME': '/var/spool/cwl',
 849                         'TMPDIR': '/tmp'
 850                     },
 851                     'name': 'test_secrets',
 852                     'runtime_constraints': {
 853                         'vcpus': 1,
 854                         'ram': 268435456
 855                     },
 856                     'use_existing': True,
 857                     'priority': 500,
 858                     'mounts': {
 859                         '/tmp': {'kind': 'tmp',
 860                                  "capacity": 1073741824
 861                              },
 862                         '/var/spool/cwl': {'kind': 'tmp',
 863                                            "capacity": 1073741824 }
 864                     },
 865                     'state': 'Committed',
 866                     'output_name': 'Output from step test_secrets',
 867                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 868                     'output_path': '/var/spool/cwl',
 869                     'output_ttl': 0,
 870                     'container_image': '99999999999999999999999999999993+99',
 871                     'command': ['md5sum', 'example.conf'],
 872                     'cwd': '/var/spool/cwl',
 873                     'scheduling_parameters': {},
 874                     'properties': {'cwl_input': job_order},
 875                     "secret_mounts": {
 876                         "/var/spool/cwl/example.conf": {
 877                             "content": "username: user\npassword: blorp\n",
 878                             "kind": "text"
 879                         }
 880                     },
 881                     'output_storage_classes': ["default"]
 882                 }))
 883
 884     # The test passes no builder.resources
 885     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 886     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 887     def test_timelimit(self, keepdocker):
 888         runner = mock.MagicMock()
 889         runner.ignore_docker_for_reuse = False
 890         runner.intermediate_output_ttl = 0
 891         runner.secret_store = cwltool.secrets.SecretStore()
 892         runner.api._rootDesc = {"revision": "20210628"}
 893         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 894
 895         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 896         runner.api.collections().get().execute.return_value = {
 897             "portable_data_hash": "99999999999999999999999999999993+99"}
 898
 899         tool = cmap({
 900             "inputs": [],
 901             "outputs": [],
 902             "baseCommand": "ls",
 903             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 904             "id": "",
 905             "cwlVersion": "v1.2",
 906             "class": "CommandLineTool",
 907             "hints": [
 908                 {
 909                     "class": "ToolTimeLimit",
 910                     "timelimit": 42
 911                 }
 912             ]
 913         })
 914
 915         loadingContext, runtimeContext = self.helper(runner)
 916         runtimeContext.name = "test_timelimit"
 917
 918         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 919         arvtool.formatgraph = None
 920
 921         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 922             j.run(runtimeContext)
 923
 924         _, kwargs = runner.api.container_requests().create.call_args
 925         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 926
 927
 928     # The test passes no builder.resources
 929     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 930     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 931     def test_setting_storage_class(self, keepdocker):
 932         #arv_docker_clear_cache()
 933
 934         runner = mock.MagicMock()
 935         runner.ignore_docker_for_reuse = False
 936         runner.intermediate_output_ttl = 0
 937         runner.secret_store = cwltool.secrets.SecretStore()
 938         runner.api._rootDesc = {"revision": "20210628"}
 939         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 940
 941         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 942         runner.api.collections().get().execute.return_value = {
 943             "portable_data_hash": "99999999999999999999999999999993+99"}
 944
 945         tool = cmap({
 946             "inputs": [],
 947             "outputs": [],
 948             "baseCommand": "ls",
 949             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 950             "id": "",
 951             "cwlVersion": "v1.2",
 952             "class": "CommandLineTool",
 953             "hints": [
 954                 {
 955                     "class": "http://arvados.org/cwl#OutputStorageClass",
 956                     "finalStorageClass": ["baz_sc", "qux_sc"],
 957                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 958                 }
 959             ]
 960         })
 961
 962         loadingContext, runtimeContext = self.helper(runner, True)
 963
 964         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 965         arvtool.formatgraph = None
 966
 967         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 968             j.run(runtimeContext)
 969             runner.api.container_requests().create.assert_called_with(
 970                 body=JsonDiffMatcher({
 971                     'environment': {
 972                         'HOME': '/var/spool/cwl',
 973                         'TMPDIR': '/tmp'
 974                     },
 975                     'name': 'test_run_True',
 976                     'runtime_constraints': {
 977                         'vcpus': 1,
 978                         'ram': 268435456
 979                     },
 980                     'use_existing': True,
 981                     'priority': 500,
 982                     'mounts': {
 983                         '/tmp': {'kind': 'tmp',
 984                                  "capacity": 1073741824
 985                              },
 986                         '/var/spool/cwl': {'kind': 'tmp',
 987                                            "capacity": 1073741824 }
 988                     },
 989                     'state': 'Committed',
 990                     'output_name': 'Output from step test_run_True',
 991                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 992                     'output_path': '/var/spool/cwl',
 993                     'output_ttl': 0,
 994                     'container_image': '99999999999999999999999999999993+99',
 995                     'command': ['ls', '/var/spool/cwl'],
 996                     'cwd': '/var/spool/cwl',
 997                     'scheduling_parameters': {},
 998                     'properties': {'cwl_input': {}},
 999                     'secret_mounts': {},
1000                     'output_storage_classes': ["foo_sc", "bar_sc"]
1001                 }))
1002
1003
1004     # The test passes no builder.resources
1005     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1006     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1007     def test_setting_process_properties(self, keepdocker):
1008         #arv_docker_clear_cache()
1009
1010         runner = mock.MagicMock()
1011         runner.ignore_docker_for_reuse = False
1012         runner.intermediate_output_ttl = 0
1013         runner.secret_store = cwltool.secrets.SecretStore()
1014         runner.api._rootDesc = {"revision": "20210628"}
1015         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1016
1017         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1018         runner.api.collections().get().execute.return_value = {
1019             "portable_data_hash": "99999999999999999999999999999993+99"}
1020
1021         tool = cmap({
1022             "inputs": [
1023                 {"id": "x", "type": "string"}],
1024             "outputs": [],
1025             "baseCommand": "ls",
1026             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1027             "id": "",
1028             "class": "CommandLineTool",
1029             "cwlVersion": "v1.2",
1030             "hints": [
1031             {
1032                 "class": "http://arvados.org/cwl#ProcessProperties",
1033                 "processProperties": [
1034                     {"propertyName": "foo",
1035                      "propertyValue": "bar"},
1036                     {"propertyName": "baz",
1037                      "propertyValue": "$(inputs.x)"},
1038                     {"propertyName": "quux",
1039                      "propertyValue": {
1040                          "q1": 1,
1041                          "q2": 2
1042                      }
1043                     }
1044                 ],
1045             }
1046         ]
1047         })
1048
1049         loadingContext, runtimeContext = self.helper(runner, True)
1050
1051         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1052         arvtool.formatgraph = None
1053
1054         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1055             j.run(runtimeContext)
1056             runner.api.container_requests().create.assert_called_with(
1057                 body=JsonDiffMatcher({
1058                     'environment': {
1059                         'HOME': '/var/spool/cwl',
1060                         'TMPDIR': '/tmp'
1061                     },
1062                     'name': 'test_run_True',
1063                     'runtime_constraints': {
1064                         'vcpus': 1,
1065                         'ram': 268435456
1066                     },
1067                     'use_existing': True,
1068                     'priority': 500,
1069                     'mounts': {
1070                         '/tmp': {'kind': 'tmp',
1071                                  "capacity": 1073741824
1072                              },
1073                         '/var/spool/cwl': {'kind': 'tmp',
1074                                            "capacity": 1073741824 }
1075                     },
1076                     'state': 'Committed',
1077                     'output_name': 'Output from step test_run_True',
1078                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1079                     'output_path': '/var/spool/cwl',
1080                     'output_ttl': 0,
1081                     'container_image': '99999999999999999999999999999993+99',
1082                     'command': ['ls', '/var/spool/cwl'],
1083                     'cwd': '/var/spool/cwl',
1084                     'scheduling_parameters': {},
1085                     'properties': {
1086                         "baz": "blorp",
1087                         "cwl_input": {"x": "blorp"},
1088                         "foo": "bar",
1089                         "quux": {
1090                             "q1": 1,
1091                             "q2": 2
1092                         }
1093                     },
1094                     'secret_mounts': {},
1095                     'output_storage_classes': ["default"]
1096                 }))
1097
1098
1099     # The test passes no builder.resources
1100     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1101     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1102     def test_cuda_requirement(self, keepdocker):
1103         arvados_cwl.add_arv_hints()
1104         #arv_docker_clear_cache()
1105
1106         runner = mock.MagicMock()
1107         runner.ignore_docker_for_reuse = False
1108         runner.intermediate_output_ttl = 0
1109         runner.secret_store = cwltool.secrets.SecretStore()
1110         runner.api._rootDesc = {"revision": "20210628"}
1111         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1112
1113         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1114         runner.api.collections().get().execute.return_value = {
1115             "portable_data_hash": "99999999999999999999999999999993+99"}
1116
1117         test_cwl_req = [{
1118                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1119                 "cudaVersionMin": "11.0",
1120                 "cudaComputeCapability": "9.0",
1121             }, {
1122                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1123                 "cudaVersionMin": "11.0",
1124                 "cudaComputeCapability": "9.0",
1125                 "cudaDeviceCountMin": 2
1126             }, {
1127                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1128                 "cudaVersionMin": "11.0",
1129                 "cudaComputeCapability": ["4.0", "5.0"],
1130                 "cudaDeviceCountMin": 2
1131             }]
1132
1133         test_arv_req = [{
1134             'device_count': 1,
1135             'driver_version': "11.0",
1136             'hardware_capability': "9.0"
1137         }, {
1138             'device_count': 2,
1139             'driver_version': "11.0",
1140             'hardware_capability': "9.0"
1141         }, {
1142             'device_count': 2,
1143             'driver_version': "11.0",
1144             'hardware_capability': "4.0"
1145         }]
1146
1147         for test_case in range(0, len(test_cwl_req)):
1148
1149             tool = cmap({
1150                 "inputs": [],
1151                 "outputs": [],
1152                 "baseCommand": "nvidia-smi",
1153                 "arguments": [],
1154                 "id": "",
1155                 "cwlVersion": "v1.2",
1156                 "class": "CommandLineTool",
1157                 "requirements": [test_cwl_req[test_case]]
1158             })
1159
1160             loadingContext, runtimeContext = self.helper(runner, True)
1161
1162             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1163             arvtool.formatgraph = None
1164
1165             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1166                 j.run(runtimeContext)
1167                 runner.api.container_requests().create.assert_called_with(
1168                     body=JsonDiffMatcher({
1169                         'environment': {
1170                             'HOME': '/var/spool/cwl',
1171                             'TMPDIR': '/tmp'
1172                         },
1173                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1174                         'runtime_constraints': {
1175                             'vcpus': 1,
1176                             'ram': 268435456,
1177                             'cuda': test_arv_req[test_case]
1178                         },
1179                         'use_existing': True,
1180                         'priority': 500,
1181                         'mounts': {
1182                             '/tmp': {'kind': 'tmp',
1183                                      "capacity": 1073741824
1184                                  },
1185                             '/var/spool/cwl': {'kind': 'tmp',
1186                                                "capacity": 1073741824 }
1187                         },
1188                         'state': 'Committed',
1189                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1190                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1191                         'output_path': '/var/spool/cwl',
1192                         'output_ttl': 0,
1193                         'container_image': '99999999999999999999999999999993+99',
1194                         'command': ['nvidia-smi'],
1195                         'cwd': '/var/spool/cwl',
1196                         'scheduling_parameters': {},
1197                         'properties': {'cwl_input': {}},
1198                         'secret_mounts': {},
1199                         'output_storage_classes': ["default"]
1200                     }))
1201
1202
1203     # The test passes no builder.resources
1204     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1205     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1206     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1207     def test_match_local_docker(self, keepdocker, determine_image_id):
1208         arvados_cwl.add_arv_hints()
1209
1210         runner = mock.MagicMock()
1211         runner.ignore_docker_for_reuse = False
1212         runner.intermediate_output_ttl = 0
1213         runner.secret_store = cwltool.secrets.SecretStore()
1214         runner.api._rootDesc = {"revision": "20210628"}
1215         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1216
1217         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1218                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1219         determine_image_id.side_effect = lambda x: "123"
1220         def execute(uuid):
1221             ex = mock.MagicMock()
1222             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1223                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1224             ex.execute.return_value = lookup[uuid]
1225             return ex
1226         runner.api.collections().get.side_effect = execute
1227
1228         tool = cmap({
1229             "inputs": [],
1230             "outputs": [],
1231             "baseCommand": "echo",
1232             "arguments": [],
1233             "id": "",
1234             "cwlVersion": "v1.0",
1235             "class": "org.w3id.cwl.cwl.CommandLineTool"
1236         })
1237
1238         loadingContext, runtimeContext = self.helper(runner, True)
1239
1240         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1241         arvtool.formatgraph = None
1242
1243         container_request = {
1244             'environment': {
1245                 'HOME': '/var/spool/cwl',
1246                 'TMPDIR': '/tmp'
1247             },
1248             'name': 'test_run_True',
1249             'runtime_constraints': {
1250                 'vcpus': 1,
1251                 'ram': 1073741824,
1252             },
1253             'use_existing': True,
1254             'priority': 500,
1255             'mounts': {
1256                 '/tmp': {'kind': 'tmp',
1257                          "capacity": 1073741824
1258                          },
1259                 '/var/spool/cwl': {'kind': 'tmp',
1260                                    "capacity": 1073741824 }
1261             },
1262             'state': 'Committed',
1263             'output_name': 'Output from step test_run_True',
1264             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1265             'output_path': '/var/spool/cwl',
1266             'output_ttl': 0,
1267             'container_image': '99999999999999999999999999999994+99',
1268             'command': ['echo'],
1269             'cwd': '/var/spool/cwl',
1270             'scheduling_parameters': {},
1271             'properties': {'cwl_input': {}},
1272             'secret_mounts': {},
1273             'output_storage_classes': ["default"]
1274         }
1275
1276         runtimeContext.match_local_docker = False
1277         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1278             j.run(runtimeContext)
1279             runner.api.container_requests().create.assert_called_with(
1280                 body=JsonDiffMatcher(container_request))
1281
1282         runtimeContext.cached_docker_lookups.clear()
1283         runtimeContext.match_local_docker = True
1284         container_request['container_image'] = '99999999999999999999999999999993+99'
1285         container_request['name'] = 'test_run_True_2'
1286         container_request['output_name'] = 'Output from step test_run_True_2'
1287         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1288             j.run(runtimeContext)
1289             runner.api.container_requests().create.assert_called_with(
1290                 body=JsonDiffMatcher(container_request))
1291
1292
1293     # The test passes no builder.resources
1294     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1296     def test_run_preemptible_hint(self, keepdocker):
1297         arvados_cwl.add_arv_hints()
1298         for enable_preemptible in (None, True, False):
1299             for preemptible_hint in (None, True, False):
1300                 #arv_docker_clear_cache()
1301
1302                 runner = mock.MagicMock()
1303                 runner.ignore_docker_for_reuse = False
1304                 runner.intermediate_output_ttl = 0
1305                 runner.secret_store = cwltool.secrets.SecretStore()
1306                 runner.api._rootDesc = {"revision": "20210628"}
1307                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1308
1309                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1310                 runner.api.collections().get().execute.return_value = {
1311                     "portable_data_hash": "99999999999999999999999999999993+99"}
1312
1313                 if preemptible_hint is not None:
1314                     hints = [{
1315                         "class": "http://arvados.org/cwl#UsePreemptible",
1316                         "usePreemptible": preemptible_hint
1317                     }]
1318                 else:
1319                     hints = []
1320
1321                 tool = cmap({
1322                     "inputs": [],
1323                     "outputs": [],
1324                     "baseCommand": "ls",
1325                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1326                     "id": "",
1327                     "class": "CommandLineTool",
1328                     "cwlVersion": "v1.2",
1329                     "hints": hints
1330                 })
1331
1332                 loadingContext, runtimeContext = self.helper(runner)
1333
1334                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1335                 runtimeContext.enable_preemptible = enable_preemptible
1336
1337                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1338                 arvtool.formatgraph = None
1339
1340                 # Test the interactions between --enable/disable-preemptible
1341                 # and UsePreemptible hint
1342
1343                 if enable_preemptible is None:
1344                     if preemptible_hint is None:
1345                         sched = {}
1346                     else:
1347                         sched = {'preemptible': preemptible_hint}
1348                 else:
1349                     if preemptible_hint is None:
1350                         sched = {'preemptible': enable_preemptible}
1351                     else:
1352                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1353
1354                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1355                     j.run(runtimeContext)
1356                     runner.api.container_requests().create.assert_called_with(
1357                         body=JsonDiffMatcher({
1358                             'environment': {
1359                                 'HOME': '/var/spool/cwl',
1360                                 'TMPDIR': '/tmp'
1361                             },
1362                             'name': runtimeContext.name,
1363                             'runtime_constraints': {
1364                                 'vcpus': 1,
1365                                 'ram': 268435456
1366                             },
1367                             'use_existing': True,
1368                             'priority': 500,
1369                             'mounts': {
1370                                 '/tmp': {'kind': 'tmp',
1371                                          "capacity": 1073741824
1372                                      },
1373                                 '/var/spool/cwl': {'kind': 'tmp',
1374                                                    "capacity": 1073741824 }
1375                             },
1376                             'state': 'Committed',
1377                             'output_name': 'Output from step '+runtimeContext.name,
1378                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1379                             'output_path': '/var/spool/cwl',
1380                             'output_ttl': 0,
1381                             'container_image': '99999999999999999999999999999993+99',
1382                             'command': ['ls', '/var/spool/cwl'],
1383                             'cwd': '/var/spool/cwl',
1384                             'scheduling_parameters': sched,
1385                             'properties': {'cwl_input': {}},
1386                             'secret_mounts': {},
1387                             'output_storage_classes': ["default"]
1388                         }))
1389
1390
1391     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1392     def test_output_properties(self, keepdocker):
1393         arvados_cwl.add_arv_hints()
1394         for rev in ["20210628", "20220510"]:
1395             runner = mock.MagicMock()
1396             runner.ignore_docker_for_reuse = False
1397             runner.intermediate_output_ttl = 0
1398             runner.secret_store = cwltool.secrets.SecretStore()
1399             runner.api._rootDesc = {"revision": rev}
1400             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1401
1402             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1403             runner.api.collections().get().execute.return_value = {
1404                 "portable_data_hash": "99999999999999999999999999999993+99"}
1405
1406             tool = cmap({
1407                 "inputs": [{
1408                     "id": "inp",
1409                     "type": "string"
1410                 }],
1411                 "outputs": [],
1412                 "baseCommand": "ls",
1413                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1414                 "id": "",
1415                 "cwlVersion": "v1.2",
1416                 "class": "CommandLineTool",
1417                 "hints": [
1418                     {
1419                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1420                         "outputProperties": {
1421                             "foo": "bar",
1422                             "baz": "$(inputs.inp)"
1423                         }
1424                     }
1425                 ]
1426             })
1427
1428             loadingContext, runtimeContext = self.helper(runner)
1429             runtimeContext.name = "test_timelimit"
1430
1431             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1432             arvtool.formatgraph = None
1433
1434             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1435                 j.run(runtimeContext)
1436
1437             _, kwargs = runner.api.container_requests().create.call_args
1438             if rev == "20220510":
1439                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1440             else:
1441                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1442
1443
1444 class TestWorkflow(unittest.TestCase):
1445     def setUp(self):
1446         cwltool.process._names = set()
1447         #arv_docker_clear_cache()
1448
1449     def helper(self, runner, enable_reuse=True):
1450         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1451
1452         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1453                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1454
1455         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1456         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1457         document_loader.fetch_text = document_loader.fetcher.fetch_text
1458         document_loader.check_exists = document_loader.fetcher.check_exists
1459
1460         loadingContext = arvados_cwl.context.ArvLoadingContext(
1461             {"avsc_names": avsc_names,
1462              "basedir": "",
1463              "make_fs_access": make_fs_access,
1464              "loader": document_loader,
1465              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1466              "construct_tool_object": runner.arv_make_tool})
1467         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1468             {"work_api": "containers",
1469              "basedir": "",
1470              "name": "test_run_wf_"+str(enable_reuse),
1471              "make_fs_access": make_fs_access,
1472              "tmpdir": "/tmp",
1473              "enable_reuse": enable_reuse,
1474              "priority": 500})
1475
1476         return loadingContext, runtimeContext
1477
1478     # The test passes no builder.resources
1479     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1480     @mock.patch("arvados.collection.CollectionReader")
1481     @mock.patch("arvados.collection.Collection")
1482     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1483     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1484         arvados_cwl.add_arv_hints()
1485
1486         api = mock.MagicMock()
1487         api._rootDesc = get_rootDesc()
1488         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1489
1490         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1491         self.assertEqual(runner.work_api, 'containers')
1492
1493         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1494         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1495         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1496                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1497
1498         runner.api.containers().current().execute.return_value = {}
1499
1500         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1501         runner.ignore_docker_for_reuse = False
1502         runner.num_retries = 0
1503         runner.secret_store = cwltool.secrets.SecretStore()
1504
1505         loadingContext, runtimeContext = self.helper(runner)
1506         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1507
1508         mockcollectionreader().exists.return_value = True
1509
1510         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1511         metadata["cwlVersion"] = tool["cwlVersion"]
1512
1513         mockc = mock.MagicMock()
1514         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1515         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1516
1517         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1518         arvtool.formatgraph = None
1519         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1520
1521         next(it).run(runtimeContext)
1522         next(it).run(runtimeContext)
1523
1524         with open("tests/wf/scatter2_subwf.cwl") as f:
1525             subwf = StripYAMLComments(f.read()).rstrip()
1526
1527         runner.api.container_requests().create.assert_called_with(
1528             body=JsonDiffMatcher({
1529                 "command": [
1530                     "cwltool",
1531                     "--no-container",
1532                     "--move-outputs",
1533                     "--preserve-entire-environment",
1534                     "workflow.cwl",
1535                     "cwl.input.yml"
1536                 ],
1537                 "container_image": "99999999999999999999999999999993+99",
1538                 "cwd": "/var/spool/cwl",
1539                 "environment": {
1540                     "HOME": "/var/spool/cwl",
1541                     "TMPDIR": "/tmp"
1542                 },
1543                 "mounts": {
1544                     "/keep/99999999999999999999999999999999+118": {
1545                         "kind": "collection",
1546                         "portable_data_hash": "99999999999999999999999999999999+118"
1547                     },
1548                     "/tmp": {
1549                         "capacity": 1073741824,
1550                         "kind": "tmp"
1551                     },
1552                     "/var/spool/cwl": {
1553                         "capacity": 1073741824,
1554                         "kind": "tmp"
1555                     },
1556                     "/var/spool/cwl/cwl.input.yml": {
1557                         "kind": "collection",
1558                         "path": "cwl.input.yml",
1559                         "portable_data_hash": "99999999999999999999999999999996+99"
1560                     },
1561                     "/var/spool/cwl/workflow.cwl": {
1562                         "kind": "collection",
1563                         "path": "workflow.cwl",
1564                         "portable_data_hash": "99999999999999999999999999999996+99"
1565                     },
1566                     "stdout": {
1567                         "kind": "file",
1568                         "path": "/var/spool/cwl/cwl.output.json"
1569                     }
1570                 },
1571                 "name": "scatterstep",
1572                 "output_name": "Output from step scatterstep",
1573                 "output_path": "/var/spool/cwl",
1574                 "output_ttl": 0,
1575                 "priority": 500,
1576                 "properties": {'cwl_input': {
1577                         "fileblub": {
1578                             "basename": "token.txt",
1579                             "class": "File",
1580                             "dirname": "/keep/99999999999999999999999999999999+118",
1581                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1582                             "nameext": ".txt",
1583                             "nameroot": "token",
1584                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1585                             "size": 0
1586                         },
1587                         "sleeptime": 5
1588                 }},
1589                 "runtime_constraints": {
1590                     "ram": 1073741824,
1591                     "vcpus": 1
1592                 },
1593                 "scheduling_parameters": {},
1594                 "secret_mounts": {},
1595                 "state": "Committed",
1596                 "use_existing": True,
1597                 'output_storage_classes': ["default"]
1598             }))
1599         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1600         mockc.open().__enter__().write.assert_has_calls([mock.call(
1601 '''{
1602   "fileblub": {
1603     "basename": "token.txt",
1604     "class": "File",
1605     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1606     "size": 0
1607   },
1608   "sleeptime": 5
1609 }''')])
1610
1611     # The test passes no builder.resources
1612     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1613     @mock.patch("arvados.collection.CollectionReader")
1614     @mock.patch("arvados.collection.Collection")
1615     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1616     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1617         arvados_cwl.add_arv_hints()
1618
1619         api = mock.MagicMock()
1620         api._rootDesc = get_rootDesc()
1621         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1622
1623         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1624         self.assertEqual(runner.work_api, 'containers')
1625
1626         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1627         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1628                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1629         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1630                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1631
1632         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1633         runner.ignore_docker_for_reuse = False
1634         runner.num_retries = 0
1635         runner.secret_store = cwltool.secrets.SecretStore()
1636
1637         loadingContext, runtimeContext = self.helper(runner)
1638         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1639         loadingContext.do_update = True
1640         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1641
1642         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1643
1644         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1645         arvtool.formatgraph = None
1646         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1647
1648         next(it).run(runtimeContext)
1649         next(it).run(runtimeContext)
1650
1651         with open("tests/wf/echo-subwf.cwl") as f:
1652             subwf = StripYAMLComments(f.read())
1653
1654         runner.api.container_requests().create.assert_called_with(
1655             body=JsonDiffMatcher({
1656                 'output_ttl': 0,
1657                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1658                 'scheduling_parameters': {},
1659                 'name': u'echo-subwf',
1660                 'secret_mounts': {},
1661                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1662                 'properties': {'cwl_input': {}},
1663                 'priority': 500,
1664                 'mounts': {
1665                     '/var/spool/cwl/cwl.input.yml': {
1666                         'portable_data_hash': '99999999999999999999999999999996+99',
1667                         'kind': 'collection',
1668                         'path': 'cwl.input.yml'
1669                     },
1670                     '/var/spool/cwl/workflow.cwl': {
1671                         'portable_data_hash': '99999999999999999999999999999996+99',
1672                         'kind': 'collection',
1673                         'path': 'workflow.cwl'
1674                     },
1675                     'stdout': {
1676                         'path': '/var/spool/cwl/cwl.output.json',
1677                         'kind': 'file'
1678                     },
1679                     '/tmp': {
1680                         'kind': 'tmp',
1681                         'capacity': 1073741824
1682                     }, '/var/spool/cwl': {
1683                         'kind': 'tmp',
1684                         'capacity': 3221225472
1685                     }
1686                 },
1687                 'state': 'Committed',
1688                 'output_path': '/var/spool/cwl',
1689                 'container_image': '99999999999999999999999999999993+99',
1690                 'command': [
1691                     u'cwltool',
1692                     u'--no-container',
1693                     u'--move-outputs',
1694                     u'--preserve-entire-environment',
1695                     u'workflow.cwl',
1696                     u'cwl.input.yml'
1697                 ],
1698                 'use_existing': True,
1699                 'output_name': u'Output from step echo-subwf',
1700                 'cwd': '/var/spool/cwl',
1701                 'output_storage_classes': ["default"]
1702             }))
1703
1704     def test_default_work_api(self):
1705         arvados_cwl.add_arv_hints()
1706
1707         api = mock.MagicMock()
1708         api._rootDesc = copy.deepcopy(get_rootDesc())
1709         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1710         self.assertEqual(runner.work_api, 'containers')