sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {'cwl_input': {}},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         tool = cmap({
 210             "inputs": [],
 211             "outputs": [],
 212             "hints": [{
 213                 "class": "ResourceRequirement",
 214                 "coresMin": 3,
 215                 "ramMin": 3000,
 216                 "tmpdirMin": 4000,
 217                 "outdirMin": 5000
 218             }, {
 219                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 220                 "keep_cache": 512
 221             }, {
 222                 "class": "http://arvados.org/cwl#APIRequirement",
 223             }, {
 224                 "class": "http://arvados.org/cwl#PartitionRequirement",
 225                 "partition": "blurb"
 226             }, {
 227                 "class": "http://arvados.org/cwl#IntermediateOutput",
 228                 "outputTTL": 7200
 229             }, {
 230                 "class": "WorkReuse",
 231                 "enableReuse": False
 232             }],
 233             "baseCommand": "ls",
 234             "id": "",
 235             "class": "CommandLineTool",
 236             "cwlVersion": "v1.2"
 237         })
 238
 239         loadingContext, runtimeContext = self.helper(runner)
 240         runtimeContext.name = "test_resource_requirements"
 241
 242         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 243         arvtool.formatgraph = None
 244         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 245             j.run(runtimeContext)
 246
 247         call_args, call_kwargs = runner.api.container_requests().create.call_args
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_resource_requirements',
 255             'runtime_constraints': {
 256                 'vcpus': 3,
 257                 'ram': 3145728000,
 258                 'keep_cache_ram': 536870912,
 259                 'API': True
 260             },
 261             'use_existing': False,
 262             'priority': 500,
 263             'mounts': {
 264                 '/tmp': {'kind': 'tmp',
 265                          "capacity": 4194304000 },
 266                 '/var/spool/cwl': {'kind': 'tmp',
 267                                    "capacity": 5242880000 }
 268             },
 269             'state': 'Committed',
 270             'output_name': 'Output from step test_resource_requirements',
 271             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272             'output_path': '/var/spool/cwl',
 273             'output_ttl': 7200,
 274             'container_image': '99999999999999999999999999999993+99',
 275             'command': ['ls'],
 276             'cwd': '/var/spool/cwl',
 277             'scheduling_parameters': {
 278                 'partitions': ['blurb']
 279             },
 280             'properties': {'cwl_input': {}},
 281             'secret_mounts': {},
 282             'output_storage_classes': ["default"]
 283         }
 284
 285         call_body = call_kwargs.get('body', None)
 286         self.assertNotEqual(None, call_body)
 287         for key in call_body:
 288             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 289
 290
 291     # The test passes some fields in builder.resources
 292     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 293     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 294     @mock.patch("arvados.collection.Collection")
 295     def test_initial_work_dir(self, collection_mock, keepdocker):
 296         runner = mock.MagicMock()
 297         runner.ignore_docker_for_reuse = False
 298         runner.intermediate_output_ttl = 0
 299         runner.secret_store = cwltool.secrets.SecretStore()
 300         runner.api._rootDesc = {"revision": "20210628"}
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         sourcemock = mock.MagicMock()
 307         def get_collection_mock(p):
 308             if "/" in p:
 309                 return (sourcemock, p.split("/", 1)[1])
 310             else:
 311                 return (sourcemock, "")
 312         runner.fs_access.get_collection.side_effect = get_collection_mock
 313
 314         vwdmock = mock.MagicMock()
 315         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 316
 317         tool = cmap({
 318             "inputs": [],
 319             "outputs": [],
 320             "hints": [{
 321                 "class": "InitialWorkDirRequirement",
 322                 "listing": [{
 323                     "class": "File",
 324                     "basename": "foo",
 325                     "location": "keep:99999999999999999999999999999995+99/bar"
 326                 },
 327                 {
 328                     "class": "Directory",
 329                     "basename": "foo2",
 330                     "location": "keep:99999999999999999999999999999995+99"
 331                 },
 332                 {
 333                     "class": "File",
 334                     "basename": "filename",
 335                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 336                 },
 337                 {
 338                     "class": "Directory",
 339                     "basename": "subdir",
 340                     "location": "keep:99999999999999999999999999999995+99/subdir"
 341                 }                        ]
 342             }],
 343             "baseCommand": "ls",
 344             "class": "CommandLineTool",
 345             "cwlVersion": "v1.2",
 346             "id": ""
 347         })
 348
 349         loadingContext, runtimeContext = self.helper(runner)
 350         runtimeContext.name = "test_initial_work_dir"
 351
 352         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 353
 354         arvtool.formatgraph = None
 355         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 356             j.run(runtimeContext)
 357
 358         call_args, call_kwargs = runner.api.container_requests().create.call_args
 359
 360         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 361         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 364
 365         call_body_expected = {
 366             'environment': {
 367                 'HOME': '/var/spool/cwl',
 368                 'TMPDIR': '/tmp'
 369             },
 370             'name': 'test_initial_work_dir',
 371             'runtime_constraints': {
 372                 'vcpus': 1,
 373                 'ram': 268435456
 374             },
 375             'use_existing': True,
 376             'priority': 500,
 377             'mounts': {
 378                 '/tmp': {'kind': 'tmp',
 379                          "capacity": 1073741824 },
 380                 '/var/spool/cwl': {'kind': 'tmp',
 381                                    "capacity": 1073741824 },
 382                 '/var/spool/cwl/foo': {
 383                     'kind': 'collection',
 384                     'path': 'foo',
 385                     'portable_data_hash': '99999999999999999999999999999996+99'
 386                 },
 387                 '/var/spool/cwl/foo2': {
 388                     'kind': 'collection',
 389                     'path': 'foo2',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/filename': {
 393                     'kind': 'collection',
 394                     'path': 'filename',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/subdir': {
 398                     'kind': 'collection',
 399                     'path': 'subdir',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 }
 402             },
 403             'state': 'Committed',
 404             'output_name': 'Output from step test_initial_work_dir',
 405             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 406             'output_path': '/var/spool/cwl',
 407             'output_ttl': 0,
 408             'container_image': '99999999999999999999999999999993+99',
 409             'command': ['ls'],
 410             'cwd': '/var/spool/cwl',
 411             'scheduling_parameters': {
 412             },
 413             'properties': {'cwl_input': {}},
 414             'secret_mounts': {},
 415             'output_storage_classes': ["default"]
 416         }
 417
 418         call_body = call_kwargs.get('body', None)
 419         self.assertNotEqual(None, call_body)
 420         for key in call_body:
 421             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 422
 423
 424     # Test redirecting stdin/stdout/stderr
 425     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 426     def test_redirects(self, keepdocker):
 427         runner = mock.MagicMock()
 428         runner.ignore_docker_for_reuse = False
 429         runner.intermediate_output_ttl = 0
 430         runner.secret_store = cwltool.secrets.SecretStore()
 431         runner.api._rootDesc = {"revision": "20210628"}
 432
 433         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 434         runner.api.collections().get().execute.return_value = {
 435             "portable_data_hash": "99999999999999999999999999999993+99"}
 436
 437         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 438
 439         tool = cmap({
 440             "inputs": [],
 441             "outputs": [],
 442             "baseCommand": "ls",
 443             "stdout": "stdout.txt",
 444             "stderr": "stderr.txt",
 445             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 446             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 447             "id": "",
 448             "class": "CommandLineTool",
 449             "cwlVersion": "v1.2"
 450         })
 451
 452         loadingContext, runtimeContext = self.helper(runner)
 453         runtimeContext.name = "test_run_redirect"
 454
 455         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 456         arvtool.formatgraph = None
 457         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 458             j.run(runtimeContext)
 459             runner.api.container_requests().create.assert_called_with(
 460                 body=JsonDiffMatcher({
 461                     'environment': {
 462                         'HOME': '/var/spool/cwl',
 463                         'TMPDIR': '/tmp'
 464                     },
 465                     'name': 'test_run_redirect',
 466                     'runtime_constraints': {
 467                         'vcpus': 1,
 468                         'ram': 268435456
 469                     },
 470                     'use_existing': True,
 471                     'priority': 500,
 472                     'mounts': {
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 },
 477                         "stderr": {
 478                             "kind": "file",
 479                             "path": "/var/spool/cwl/stderr.txt"
 480                         },
 481                         "stdin": {
 482                             "kind": "collection",
 483                             "path": "file.txt",
 484                             "portable_data_hash": "99999999999999999999999999999996+99"
 485                         },
 486                         "stdout": {
 487                             "kind": "file",
 488                             "path": "/var/spool/cwl/stdout.txt"
 489                         },
 490                     },
 491                     'state': 'Committed',
 492                     "output_name": "Output from step test_run_redirect",
 493                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 494                     'output_path': '/var/spool/cwl',
 495                     'output_ttl': 0,
 496                     'container_image': '99999999999999999999999999999993+99',
 497                     'command': ['ls', '/var/spool/cwl'],
 498                     'cwd': '/var/spool/cwl',
 499                     'scheduling_parameters': {},
 500                     'properties': {'cwl_input': {}},
 501                     'secret_mounts': {},
 502                     'output_storage_classes': ["default"]
 503                 }))
 504
 505     @mock.patch("arvados.collection.Collection")
 506     def test_done(self, col):
 507         api = mock.MagicMock()
 508
 509         runner = mock.MagicMock()
 510         runner.api = api
 511         runner.num_retries = 0
 512         runner.ignore_docker_for_reuse = False
 513         runner.intermediate_output_ttl = 0
 514         runner.secret_store = cwltool.secrets.SecretStore()
 515
 516         runner.api.containers().get().execute.return_value = {"state":"Complete",
 517                                                               "output": "abc+123",
 518                                                               "exit_code": 0}
 519
 520         col().open.return_value = []
 521
 522         loadingContext, runtimeContext = self.helper(runner)
 523
 524         arvjob = arvados_cwl.ArvadosContainer(runner,
 525                                               runtimeContext,
 526                                               mock.MagicMock(),
 527                                               {},
 528                                               None,
 529                                               [],
 530                                               [],
 531                                               "testjob")
 532         arvjob.output_callback = mock.MagicMock()
 533         arvjob.collect_outputs = mock.MagicMock()
 534         arvjob.successCodes = [0]
 535         arvjob.outdir = "/var/spool/cwl"
 536         arvjob.output_ttl = 3600
 537
 538         arvjob.collect_outputs.return_value = {"out": "stuff"}
 539
 540         arvjob.done({
 541             "state": "Final",
 542             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 543             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 544             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 545             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 546             "modified_at": "2017-05-26T12:01:22Z"
 547         })
 548
 549         self.assertFalse(api.collections().create.called)
 550         self.assertFalse(runner.runtime_status_error.called)
 551
 552         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 553         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 554         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 555
 556     # Test to make sure we dont call runtime_status_update if we already did
 557     # some where higher up in the call stack
 558     @mock.patch("arvados_cwl.util.get_current_container")
 559     def test_recursive_runtime_status_update(self, gcc_mock):
 560         self.setup_and_test_container_executor_and_logging(gcc_mock)
 561         root_logger = logging.getLogger('')
 562
 563         # get_current_container is invoked when we call runtime_status_update
 564         # so try and log again!
 565         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 566         try:
 567             root_logger.error("First Error")
 568         except RuntimeError:
 569             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 570
 571
 572     # Test to make sure that an exception raised from
 573     # get_current_container doesn't cause the logger to raise an
 574     # exception
 575     @mock.patch("arvados_cwl.util.get_current_container")
 576     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 577         self.setup_and_test_container_executor_and_logging(gcc_mock)
 578         root_logger = logging.getLogger('')
 579
 580         # get_current_container is invoked when we call
 581         # runtime_status_update, it is going to also raise an
 582         # exception.
 583         gcc_mock.side_effect = Exception("Second Error")
 584         try:
 585             root_logger.error("First Error")
 586         except Exception:
 587             self.fail("Exception in logger should not propagate")
 588         self.assertTrue(gcc_mock.called)
 589
 590     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 591     @mock.patch("arvados_cwl.util.get_current_container")
 592     @mock.patch("arvados.collection.CollectionReader")
 593     @mock.patch("arvados.collection.Collection")
 594     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 595         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 596
 597         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 598         self.assertTrue(gcc_mock.called)
 599
 600         runner.num_retries = 0
 601         runner.ignore_docker_for_reuse = False
 602         runner.intermediate_output_ttl = 0
 603         runner.secret_store = cwltool.secrets.SecretStore()
 604         runner.label = mock.MagicMock()
 605         runner.label.return_value = '[container testjob]'
 606
 607         runner.api.containers().get().execute.return_value = {
 608             "state":"Complete",
 609             "output": "abc+123",
 610             "exit_code": 1,
 611             "log": "def+234"
 612         }
 613
 614         col().open.return_value = []
 615
 616         loadingContext, runtimeContext = self.helper(runner)
 617
 618         arvjob = arvados_cwl.ArvadosContainer(runner,
 619                                               runtimeContext,
 620                                               mock.MagicMock(),
 621                                               {},
 622                                               None,
 623                                               [],
 624                                               [],
 625                                               "testjob")
 626         arvjob.output_callback = mock.MagicMock()
 627         arvjob.collect_outputs = mock.MagicMock()
 628         arvjob.successCodes = [0]
 629         arvjob.outdir = "/var/spool/cwl"
 630         arvjob.output_ttl = 3600
 631         arvjob.collect_outputs.return_value = {"out": "stuff"}
 632
 633         arvjob.done({
 634             "state": "Final",
 635             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 636             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 637             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 638             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 639             "modified_at": "2017-05-26T12:01:22Z"
 640         })
 641
 642         rts_mock.assert_called_with(
 643             'error',
 644             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 645             '  ** log is empty **'
 646         )
 647         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 648
 649     # The test passes no builder.resources
 650     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 651     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 652     def test_mounts(self, keepdocker):
 653         runner = mock.MagicMock()
 654         runner.ignore_docker_for_reuse = False
 655         runner.intermediate_output_ttl = 0
 656         runner.secret_store = cwltool.secrets.SecretStore()
 657         runner.api._rootDesc = {"revision": "20210628"}
 658
 659         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 660         runner.api.collections().get().execute.return_value = {
 661             "portable_data_hash": "99999999999999999999999999999994+99",
 662             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 663
 664         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 665
 666         tool = cmap({
 667             "inputs": [
 668                 {"id": "p1",
 669                  "type": "Directory"}
 670             ],
 671             "outputs": [],
 672             "baseCommand": "ls",
 673             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 674             "id": "",
 675             "class": "CommandLineTool",
 676             "cwlVersion": "v1.2"
 677         })
 678
 679         loadingContext, runtimeContext = self.helper(runner)
 680         runtimeContext.name = "test_run_mounts"
 681
 682         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 683         arvtool.formatgraph = None
 684         job_order = {
 685             "p1": {
 686                 "class": "Directory",
 687                 "location": "keep:99999999999999999999999999999994+44",
 688                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 689                 "listing": [
 690                     {
 691                         "class": "File",
 692                         "location": "keep:99999999999999999999999999999994+44/file1",
 693                     },
 694                     {
 695                         "class": "File",
 696                         "location": "keep:99999999999999999999999999999994+44/file2",
 697                     }
 698                 ]
 699             }
 700         }
 701         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 702             j.run(runtimeContext)
 703             runner.api.container_requests().create.assert_called_with(
 704                 body=JsonDiffMatcher({
 705                     'environment': {
 706                         'HOME': '/var/spool/cwl',
 707                         'TMPDIR': '/tmp'
 708                     },
 709                     'name': 'test_run_mounts',
 710                     'runtime_constraints': {
 711                         'vcpus': 1,
 712                         'ram': 268435456
 713                     },
 714                     'use_existing': True,
 715                     'priority': 500,
 716                     'mounts': {
 717                         "/keep/99999999999999999999999999999994+44": {
 718                             "kind": "collection",
 719                             "portable_data_hash": "99999999999999999999999999999994+44",
 720                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 721                         },
 722                         '/tmp': {'kind': 'tmp',
 723                                  "capacity": 1073741824 },
 724                         '/var/spool/cwl': {'kind': 'tmp',
 725                                            "capacity": 1073741824 }
 726                     },
 727                     'state': 'Committed',
 728                     'output_name': 'Output from step test_run_mounts',
 729                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 730                     'output_path': '/var/spool/cwl',
 731                     'output_ttl': 0,
 732                     'container_image': '99999999999999999999999999999994+99',
 733                     'command': ['ls', '/var/spool/cwl'],
 734                     'cwd': '/var/spool/cwl',
 735                     'scheduling_parameters': {},
 736                     'properties': {'cwl_input': {
 737                         "p1": {
 738                             "basename": "99999999999999999999999999999994+44",
 739                             "class": "Directory",
 740                             "dirname": "/keep",
 741                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 742                             "listing": [
 743                                 {
 744                                     "basename": "file1",
 745                                     "class": "File",
 746                                     "dirname": "/keep/99999999999999999999999999999994+44",
 747                                     "location": "keep:99999999999999999999999999999994+44/file1",
 748                                     "nameext": "",
 749                                     "nameroot": "file1",
 750                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 751                                     "size": 0
 752                                 },
 753                                 {
 754                                     "basename": "file2",
 755                                     "class": "File",
 756                                     "dirname": "/keep/99999999999999999999999999999994+44",
 757                                     "location": "keep:99999999999999999999999999999994+44/file2",
 758                                     "nameext": "",
 759                                     "nameroot": "file2",
 760                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 761                                     "size": 0
 762                                 }
 763                             ],
 764                             "location": "keep:99999999999999999999999999999994+44",
 765                             "path": "/keep/99999999999999999999999999999994+44"
 766                         }
 767                     }},
 768                     'secret_mounts': {},
 769                     'output_storage_classes': ["default"]
 770                 }))
 771
 772     # The test passes no builder.resources
 773     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 774     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 775     def test_secrets(self, keepdocker):
 776         arvados_cwl.add_arv_hints()
 777         runner = mock.MagicMock()
 778         runner.ignore_docker_for_reuse = False
 779         runner.intermediate_output_ttl = 0
 780         runner.secret_store = cwltool.secrets.SecretStore()
 781         runner.api._rootDesc = {"revision": "20210628"}
 782
 783         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 784         runner.api.collections().get().execute.return_value = {
 785             "portable_data_hash": "99999999999999999999999999999993+99"}
 786
 787         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 788
 789         tool = cmap({"arguments": ["md5sum", "example.conf"],
 790                      "class": "CommandLineTool",
 791                      "cwlVersion": "v1.2",
 792                      "hints": [
 793                          {
 794                              "class": "http://commonwl.org/cwltool#Secrets",
 795                              "secrets": [
 796                                  "#secret_job.cwl/pw"
 797                              ]
 798                          }
 799                      ],
 800                      "id": "",
 801                      "inputs": [
 802                          {
 803                              "id": "#secret_job.cwl/pw",
 804                              "type": "string"
 805                          }
 806                      ],
 807                      "outputs": [
 808                      ],
 809                      "requirements": [
 810                          {
 811                              "class": "InitialWorkDirRequirement",
 812                              "listing": [
 813                                  {
 814                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 815                                      "entryname": "example.conf"
 816                                  }
 817                              ]
 818                          }
 819                      ]})
 820
 821         loadingContext, runtimeContext = self.helper(runner)
 822         runtimeContext.name = "test_secrets"
 823
 824         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 825         arvtool.formatgraph = None
 826
 827         job_order = {"pw": "blorp"}
 828         runner.secret_store.store(["pw"], job_order)
 829
 830         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 831             j.run(runtimeContext)
 832             runner.api.container_requests().create.assert_called_with(
 833                 body=JsonDiffMatcher({
 834                     'environment': {
 835                         'HOME': '/var/spool/cwl',
 836                         'TMPDIR': '/tmp'
 837                     },
 838                     'name': 'test_secrets',
 839                     'runtime_constraints': {
 840                         'vcpus': 1,
 841                         'ram': 268435456
 842                     },
 843                     'use_existing': True,
 844                     'priority': 500,
 845                     'mounts': {
 846                         '/tmp': {'kind': 'tmp',
 847                                  "capacity": 1073741824
 848                              },
 849                         '/var/spool/cwl': {'kind': 'tmp',
 850                                            "capacity": 1073741824 }
 851                     },
 852                     'state': 'Committed',
 853                     'output_name': 'Output from step test_secrets',
 854                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 855                     'output_path': '/var/spool/cwl',
 856                     'output_ttl': 0,
 857                     'container_image': '99999999999999999999999999999993+99',
 858                     'command': ['md5sum', 'example.conf'],
 859                     'cwd': '/var/spool/cwl',
 860                     'scheduling_parameters': {},
 861                     'properties': {'cwl_input': job_order},
 862                     "secret_mounts": {
 863                         "/var/spool/cwl/example.conf": {
 864                             "content": "username: user\npassword: blorp\n",
 865                             "kind": "text"
 866                         }
 867                     },
 868                     'output_storage_classes': ["default"]
 869                 }))
 870
 871     # The test passes no builder.resources
 872     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 873     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 874     def test_timelimit(self, keepdocker):
 875         runner = mock.MagicMock()
 876         runner.ignore_docker_for_reuse = False
 877         runner.intermediate_output_ttl = 0
 878         runner.secret_store = cwltool.secrets.SecretStore()
 879         runner.api._rootDesc = {"revision": "20210628"}
 880
 881         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 882         runner.api.collections().get().execute.return_value = {
 883             "portable_data_hash": "99999999999999999999999999999993+99"}
 884
 885         tool = cmap({
 886             "inputs": [],
 887             "outputs": [],
 888             "baseCommand": "ls",
 889             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 890             "id": "",
 891             "cwlVersion": "v1.2",
 892             "class": "CommandLineTool",
 893             "hints": [
 894                 {
 895                     "class": "ToolTimeLimit",
 896                     "timelimit": 42
 897                 }
 898             ]
 899         })
 900
 901         loadingContext, runtimeContext = self.helper(runner)
 902         runtimeContext.name = "test_timelimit"
 903
 904         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 905         arvtool.formatgraph = None
 906
 907         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 908             j.run(runtimeContext)
 909
 910         _, kwargs = runner.api.container_requests().create.call_args
 911         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 912
 913
 914     # The test passes no builder.resources
 915     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 916     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 917     def test_setting_storage_class(self, keepdocker):
 918         arv_docker_clear_cache()
 919
 920         runner = mock.MagicMock()
 921         runner.ignore_docker_for_reuse = False
 922         runner.intermediate_output_ttl = 0
 923         runner.secret_store = cwltool.secrets.SecretStore()
 924         runner.api._rootDesc = {"revision": "20210628"}
 925
 926         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 927         runner.api.collections().get().execute.return_value = {
 928             "portable_data_hash": "99999999999999999999999999999993+99"}
 929
 930         tool = cmap({
 931             "inputs": [],
 932             "outputs": [],
 933             "baseCommand": "ls",
 934             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 935             "id": "",
 936             "cwlVersion": "v1.2",
 937             "class": "CommandLineTool",
 938             "hints": [
 939                 {
 940                     "class": "http://arvados.org/cwl#OutputStorageClass",
 941                     "finalStorageClass": ["baz_sc", "qux_sc"],
 942                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 943                 }
 944             ]
 945         })
 946
 947         loadingContext, runtimeContext = self.helper(runner, True)
 948
 949         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 950         arvtool.formatgraph = None
 951
 952         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 953             j.run(runtimeContext)
 954             runner.api.container_requests().create.assert_called_with(
 955                 body=JsonDiffMatcher({
 956                     'environment': {
 957                         'HOME': '/var/spool/cwl',
 958                         'TMPDIR': '/tmp'
 959                     },
 960                     'name': 'test_run_True',
 961                     'runtime_constraints': {
 962                         'vcpus': 1,
 963                         'ram': 268435456
 964                     },
 965                     'use_existing': True,
 966                     'priority': 500,
 967                     'mounts': {
 968                         '/tmp': {'kind': 'tmp',
 969                                  "capacity": 1073741824
 970                              },
 971                         '/var/spool/cwl': {'kind': 'tmp',
 972                                            "capacity": 1073741824 }
 973                     },
 974                     'state': 'Committed',
 975                     'output_name': 'Output from step test_run_True',
 976                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 977                     'output_path': '/var/spool/cwl',
 978                     'output_ttl': 0,
 979                     'container_image': '99999999999999999999999999999993+99',
 980                     'command': ['ls', '/var/spool/cwl'],
 981                     'cwd': '/var/spool/cwl',
 982                     'scheduling_parameters': {},
 983                     'properties': {'cwl_input': {}},
 984                     'secret_mounts': {},
 985                     'output_storage_classes': ["foo_sc", "bar_sc"]
 986                 }))
 987
 988
 989     # The test passes no builder.resources
 990     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 991     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 992     def test_setting_process_properties(self, keepdocker):
 993         arv_docker_clear_cache()
 994
 995         runner = mock.MagicMock()
 996         runner.ignore_docker_for_reuse = False
 997         runner.intermediate_output_ttl = 0
 998         runner.secret_store = cwltool.secrets.SecretStore()
 999         runner.api._rootDesc = {"revision": "20210628"}
1000
1001         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1002         runner.api.collections().get().execute.return_value = {
1003             "portable_data_hash": "99999999999999999999999999999993+99"}
1004
1005         tool = cmap({
1006             "inputs": [
1007                 {"id": "x", "type": "string"}],
1008             "outputs": [],
1009             "baseCommand": "ls",
1010             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1011             "id": "",
1012             "class": "CommandLineTool",
1013             "cwlVersion": "v1.2",
1014             "hints": [
1015             {
1016                 "class": "http://arvados.org/cwl#ProcessProperties",
1017                 "processProperties": [
1018                     {"propertyName": "foo",
1019                      "propertyValue": "bar"},
1020                     {"propertyName": "baz",
1021                      "propertyValue": "$(inputs.x)"},
1022                     {"propertyName": "quux",
1023                      "propertyValue": {
1024                          "q1": 1,
1025                          "q2": 2
1026                      }
1027                     }
1028                 ],
1029             }
1030         ]
1031         })
1032
1033         loadingContext, runtimeContext = self.helper(runner, True)
1034
1035         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1036         arvtool.formatgraph = None
1037
1038         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1039             j.run(runtimeContext)
1040             runner.api.container_requests().create.assert_called_with(
1041                 body=JsonDiffMatcher({
1042                     'environment': {
1043                         'HOME': '/var/spool/cwl',
1044                         'TMPDIR': '/tmp'
1045                     },
1046                     'name': 'test_run_True',
1047                     'runtime_constraints': {
1048                         'vcpus': 1,
1049                         'ram': 268435456
1050                     },
1051                     'use_existing': True,
1052                     'priority': 500,
1053                     'mounts': {
1054                         '/tmp': {'kind': 'tmp',
1055                                  "capacity": 1073741824
1056                              },
1057                         '/var/spool/cwl': {'kind': 'tmp',
1058                                            "capacity": 1073741824 }
1059                     },
1060                     'state': 'Committed',
1061                     'output_name': 'Output from step test_run_True',
1062                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1063                     'output_path': '/var/spool/cwl',
1064                     'output_ttl': 0,
1065                     'container_image': '99999999999999999999999999999993+99',
1066                     'command': ['ls', '/var/spool/cwl'],
1067                     'cwd': '/var/spool/cwl',
1068                     'scheduling_parameters': {},
1069                     'properties': {
1070                         "baz": "blorp",
1071                         "cwl_input": {"x": "blorp"},
1072                         "foo": "bar",
1073                         "quux": {
1074                             "q1": 1,
1075                             "q2": 2
1076                         }
1077                     },
1078                     'secret_mounts': {},
1079                     'output_storage_classes': ["default"]
1080                 }))
1081
1082
1083     # The test passes no builder.resources
1084     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1085     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1086     def test_cuda_requirement(self, keepdocker):
1087         arvados_cwl.add_arv_hints()
1088         arv_docker_clear_cache()
1089
1090         runner = mock.MagicMock()
1091         runner.ignore_docker_for_reuse = False
1092         runner.intermediate_output_ttl = 0
1093         runner.secret_store = cwltool.secrets.SecretStore()
1094         runner.api._rootDesc = {"revision": "20210628"}
1095
1096         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1097         runner.api.collections().get().execute.return_value = {
1098             "portable_data_hash": "99999999999999999999999999999993+99"}
1099
1100         test_cwl_req = [{
1101                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1102                 "cudaVersionMin": "11.0",
1103                 "cudaComputeCapability": "9.0",
1104             }, {
1105                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1106                 "cudaVersionMin": "11.0",
1107                 "cudaComputeCapability": "9.0",
1108                 "cudaDeviceCountMin": 2
1109             }, {
1110                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1111                 "cudaVersionMin": "11.0",
1112                 "cudaComputeCapability": ["4.0", "5.0"],
1113                 "cudaDeviceCountMin": 2
1114             }]
1115
1116         test_arv_req = [{
1117             'device_count': 1,
1118             'driver_version': "11.0",
1119             'hardware_capability': "9.0"
1120         }, {
1121             'device_count': 2,
1122             'driver_version': "11.0",
1123             'hardware_capability': "9.0"
1124         }, {
1125             'device_count': 2,
1126             'driver_version': "11.0",
1127             'hardware_capability': "4.0"
1128         }]
1129
1130         for test_case in range(0, len(test_cwl_req)):
1131
1132             tool = cmap({
1133                 "inputs": [],
1134                 "outputs": [],
1135                 "baseCommand": "nvidia-smi",
1136                 "arguments": [],
1137                 "id": "",
1138                 "cwlVersion": "v1.2",
1139                 "class": "CommandLineTool",
1140                 "requirements": [test_cwl_req[test_case]]
1141             })
1142
1143             loadingContext, runtimeContext = self.helper(runner, True)
1144
1145             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1146             arvtool.formatgraph = None
1147
1148             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1149                 j.run(runtimeContext)
1150                 runner.api.container_requests().create.assert_called_with(
1151                     body=JsonDiffMatcher({
1152                         'environment': {
1153                             'HOME': '/var/spool/cwl',
1154                             'TMPDIR': '/tmp'
1155                         },
1156                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1157                         'runtime_constraints': {
1158                             'vcpus': 1,
1159                             'ram': 268435456,
1160                             'cuda': test_arv_req[test_case]
1161                         },
1162                         'use_existing': True,
1163                         'priority': 500,
1164                         'mounts': {
1165                             '/tmp': {'kind': 'tmp',
1166                                      "capacity": 1073741824
1167                                  },
1168                             '/var/spool/cwl': {'kind': 'tmp',
1169                                                "capacity": 1073741824 }
1170                         },
1171                         'state': 'Committed',
1172                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1173                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1174                         'output_path': '/var/spool/cwl',
1175                         'output_ttl': 0,
1176                         'container_image': '99999999999999999999999999999993+99',
1177                         'command': ['nvidia-smi'],
1178                         'cwd': '/var/spool/cwl',
1179                         'scheduling_parameters': {},
1180                         'properties': {'cwl_input': {}},
1181                         'secret_mounts': {},
1182                         'output_storage_classes': ["default"]
1183                     }))
1184
1185
1186     # The test passes no builder.resources
1187     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1188     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1189     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1190     def test_match_local_docker(self, keepdocker, determine_image_id):
1191         arvados_cwl.add_arv_hints()
1192         arv_docker_clear_cache()
1193
1194         runner = mock.MagicMock()
1195         runner.ignore_docker_for_reuse = False
1196         runner.intermediate_output_ttl = 0
1197         runner.secret_store = cwltool.secrets.SecretStore()
1198         runner.api._rootDesc = {"revision": "20210628"}
1199
1200         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1201                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1202         determine_image_id.side_effect = lambda x: "123"
1203         def execute(uuid):
1204             ex = mock.MagicMock()
1205             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1206                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1207             ex.execute.return_value = lookup[uuid]
1208             return ex
1209         runner.api.collections().get.side_effect = execute
1210
1211         tool = cmap({
1212             "inputs": [],
1213             "outputs": [],
1214             "baseCommand": "echo",
1215             "arguments": [],
1216             "id": "",
1217             "cwlVersion": "v1.0",
1218             "class": "org.w3id.cwl.cwl.CommandLineTool"
1219         })
1220
1221         loadingContext, runtimeContext = self.helper(runner, True)
1222
1223         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1224         arvtool.formatgraph = None
1225
1226         container_request = {
1227             'environment': {
1228                 'HOME': '/var/spool/cwl',
1229                 'TMPDIR': '/tmp'
1230             },
1231             'name': 'test_run_True',
1232             'runtime_constraints': {
1233                 'vcpus': 1,
1234                 'ram': 1073741824,
1235             },
1236             'use_existing': True,
1237             'priority': 500,
1238             'mounts': {
1239                 '/tmp': {'kind': 'tmp',
1240                          "capacity": 1073741824
1241                          },
1242                 '/var/spool/cwl': {'kind': 'tmp',
1243                                    "capacity": 1073741824 }
1244             },
1245             'state': 'Committed',
1246             'output_name': 'Output from step test_run_True',
1247             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1248             'output_path': '/var/spool/cwl',
1249             'output_ttl': 0,
1250             'container_image': '99999999999999999999999999999994+99',
1251             'command': ['echo'],
1252             'cwd': '/var/spool/cwl',
1253             'scheduling_parameters': {},
1254             'properties': {'cwl_input': {}},
1255             'secret_mounts': {},
1256             'output_storage_classes': ["default"]
1257         }
1258
1259         runtimeContext.match_local_docker = False
1260         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1261             j.run(runtimeContext)
1262             runner.api.container_requests().create.assert_called_with(
1263                 body=JsonDiffMatcher(container_request))
1264
1265         arv_docker_clear_cache()
1266         runtimeContext.match_local_docker = True
1267         container_request['container_image'] = '99999999999999999999999999999993+99'
1268         container_request['name'] = 'test_run_True_2'
1269         container_request['output_name'] = 'Output from step test_run_True_2'
1270         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1271             j.run(runtimeContext)
1272             runner.api.container_requests().create.assert_called_with(
1273                 body=JsonDiffMatcher(container_request))
1274
1275
1276     # The test passes no builder.resources
1277     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1278     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1279     def test_run_preemptible_hint(self, keepdocker):
1280         arvados_cwl.add_arv_hints()
1281         for enable_preemptible in (None, True, False):
1282             for preemptible_hint in (None, True, False):
1283                 arv_docker_clear_cache()
1284
1285                 runner = mock.MagicMock()
1286                 runner.ignore_docker_for_reuse = False
1287                 runner.intermediate_output_ttl = 0
1288                 runner.secret_store = cwltool.secrets.SecretStore()
1289                 runner.api._rootDesc = {"revision": "20210628"}
1290
1291                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1292                 runner.api.collections().get().execute.return_value = {
1293                     "portable_data_hash": "99999999999999999999999999999993+99"}
1294
1295                 if preemptible_hint is not None:
1296                     hints = [{
1297                         "class": "http://arvados.org/cwl#UsePreemptible",
1298                         "usePreemptible": preemptible_hint
1299                     }]
1300                 else:
1301                     hints = []
1302
1303                 tool = cmap({
1304                     "inputs": [],
1305                     "outputs": [],
1306                     "baseCommand": "ls",
1307                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1308                     "id": "",
1309                     "class": "CommandLineTool",
1310                     "cwlVersion": "v1.2",
1311                     "hints": hints
1312                 })
1313
1314                 loadingContext, runtimeContext = self.helper(runner)
1315
1316                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1317                 runtimeContext.enable_preemptible = enable_preemptible
1318
1319                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1320                 arvtool.formatgraph = None
1321
1322                 # Test the interactions between --enable/disable-preemptible
1323                 # and UsePreemptible hint
1324
1325                 if enable_preemptible is None:
1326                     if preemptible_hint is None:
1327                         sched = {}
1328                     else:
1329                         sched = {'preemptible': preemptible_hint}
1330                 else:
1331                     if preemptible_hint is None:
1332                         sched = {'preemptible': enable_preemptible}
1333                     else:
1334                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1335
1336                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1337                     j.run(runtimeContext)
1338                     runner.api.container_requests().create.assert_called_with(
1339                         body=JsonDiffMatcher({
1340                             'environment': {
1341                                 'HOME': '/var/spool/cwl',
1342                                 'TMPDIR': '/tmp'
1343                             },
1344                             'name': runtimeContext.name,
1345                             'runtime_constraints': {
1346                                 'vcpus': 1,
1347                                 'ram': 268435456
1348                             },
1349                             'use_existing': True,
1350                             'priority': 500,
1351                             'mounts': {
1352                                 '/tmp': {'kind': 'tmp',
1353                                          "capacity": 1073741824
1354                                      },
1355                                 '/var/spool/cwl': {'kind': 'tmp',
1356                                                    "capacity": 1073741824 }
1357                             },
1358                             'state': 'Committed',
1359                             'output_name': 'Output from step '+runtimeContext.name,
1360                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1361                             'output_path': '/var/spool/cwl',
1362                             'output_ttl': 0,
1363                             'container_image': '99999999999999999999999999999993+99',
1364                             'command': ['ls', '/var/spool/cwl'],
1365                             'cwd': '/var/spool/cwl',
1366                             'scheduling_parameters': sched,
1367                             'properties': {'cwl_input': {}},
1368                             'secret_mounts': {},
1369                             'output_storage_classes': ["default"]
1370                         }))
1371
1372
1373     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1374     def test_output_properties(self, keepdocker):
1375         arvados_cwl.add_arv_hints()
1376         for rev in ["20210628", "20220510"]:
1377             runner = mock.MagicMock()
1378             runner.ignore_docker_for_reuse = False
1379             runner.intermediate_output_ttl = 0
1380             runner.secret_store = cwltool.secrets.SecretStore()
1381             runner.api._rootDesc = {"revision": rev}
1382
1383             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1384             runner.api.collections().get().execute.return_value = {
1385                 "portable_data_hash": "99999999999999999999999999999993+99"}
1386
1387             tool = cmap({
1388                 "inputs": [{
1389                     "id": "inp",
1390                     "type": "string"
1391                 }],
1392                 "outputs": [],
1393                 "baseCommand": "ls",
1394                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1395                 "id": "",
1396                 "cwlVersion": "v1.2",
1397                 "class": "CommandLineTool",
1398                 "hints": [
1399                     {
1400                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1401                         "outputProperties": {
1402                             "foo": "bar",
1403                             "baz": "$(inputs.inp)"
1404                         }
1405                     }
1406                 ]
1407             })
1408
1409             loadingContext, runtimeContext = self.helper(runner)
1410             runtimeContext.name = "test_timelimit"
1411
1412             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1413             arvtool.formatgraph = None
1414
1415             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1416                 j.run(runtimeContext)
1417
1418             _, kwargs = runner.api.container_requests().create.call_args
1419             if rev == "20220510":
1420                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1421             else:
1422                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1423
1424
1425 class TestWorkflow(unittest.TestCase):
1426     def setUp(self):
1427         cwltool.process._names = set()
1428         arv_docker_clear_cache()
1429
1430     def helper(self, runner, enable_reuse=True):
1431         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1432
1433         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1434                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1435
1436         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1437         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1438         document_loader.fetch_text = document_loader.fetcher.fetch_text
1439         document_loader.check_exists = document_loader.fetcher.check_exists
1440
1441         loadingContext = arvados_cwl.context.ArvLoadingContext(
1442             {"avsc_names": avsc_names,
1443              "basedir": "",
1444              "make_fs_access": make_fs_access,
1445              "loader": document_loader,
1446              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1447              "construct_tool_object": runner.arv_make_tool})
1448         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1449             {"work_api": "containers",
1450              "basedir": "",
1451              "name": "test_run_wf_"+str(enable_reuse),
1452              "make_fs_access": make_fs_access,
1453              "tmpdir": "/tmp",
1454              "enable_reuse": enable_reuse,
1455              "priority": 500})
1456
1457         return loadingContext, runtimeContext
1458
1459     # The test passes no builder.resources
1460     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1461     @mock.patch("arvados.collection.CollectionReader")
1462     @mock.patch("arvados.collection.Collection")
1463     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1464     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1465         arvados_cwl.add_arv_hints()
1466
1467         api = mock.MagicMock()
1468         api._rootDesc = get_rootDesc()
1469
1470         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1471         self.assertEqual(runner.work_api, 'containers')
1472
1473         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1474         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1475         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1476                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1477
1478         runner.api.containers().current().execute.return_value = {}
1479
1480         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1481         runner.ignore_docker_for_reuse = False
1482         runner.num_retries = 0
1483         runner.secret_store = cwltool.secrets.SecretStore()
1484
1485         loadingContext, runtimeContext = self.helper(runner)
1486         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1487
1488         mockcollectionreader().exists.return_value = True
1489
1490         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1491         metadata["cwlVersion"] = tool["cwlVersion"]
1492
1493         mockc = mock.MagicMock()
1494         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1495         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1496
1497         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1498         arvtool.formatgraph = None
1499         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1500
1501         next(it).run(runtimeContext)
1502         next(it).run(runtimeContext)
1503
1504         with open("tests/wf/scatter2_subwf.cwl") as f:
1505             subwf = StripYAMLComments(f.read()).rstrip()
1506
1507         runner.api.container_requests().create.assert_called_with(
1508             body=JsonDiffMatcher({
1509                 "command": [
1510                     "cwltool",
1511                     "--no-container",
1512                     "--move-outputs",
1513                     "--preserve-entire-environment",
1514                     "workflow.cwl",
1515                     "cwl.input.yml"
1516                 ],
1517                 "container_image": "99999999999999999999999999999993+99",
1518                 "cwd": "/var/spool/cwl",
1519                 "environment": {
1520                     "HOME": "/var/spool/cwl",
1521                     "TMPDIR": "/tmp"
1522                 },
1523                 "mounts": {
1524                     "/keep/99999999999999999999999999999999+118": {
1525                         "kind": "collection",
1526                         "portable_data_hash": "99999999999999999999999999999999+118"
1527                     },
1528                     "/tmp": {
1529                         "capacity": 1073741824,
1530                         "kind": "tmp"
1531                     },
1532                     "/var/spool/cwl": {
1533                         "capacity": 1073741824,
1534                         "kind": "tmp"
1535                     },
1536                     "/var/spool/cwl/cwl.input.yml": {
1537                         "kind": "collection",
1538                         "path": "cwl.input.yml",
1539                         "portable_data_hash": "99999999999999999999999999999996+99"
1540                     },
1541                     "/var/spool/cwl/workflow.cwl": {
1542                         "kind": "collection",
1543                         "path": "workflow.cwl",
1544                         "portable_data_hash": "99999999999999999999999999999996+99"
1545                     },
1546                     "stdout": {
1547                         "kind": "file",
1548                         "path": "/var/spool/cwl/cwl.output.json"
1549                     }
1550                 },
1551                 "name": "scatterstep",
1552                 "output_name": "Output from step scatterstep",
1553                 "output_path": "/var/spool/cwl",
1554                 "output_ttl": 0,
1555                 "priority": 500,
1556                 "properties": {'cwl_input': {
1557                         "fileblub": {
1558                             "basename": "token.txt",
1559                             "class": "File",
1560                             "dirname": "/keep/99999999999999999999999999999999+118",
1561                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1562                             "nameext": ".txt",
1563                             "nameroot": "token",
1564                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1565                             "size": 0
1566                         },
1567                         "sleeptime": 5
1568                 }},
1569                 "runtime_constraints": {
1570                     "ram": 1073741824,
1571                     "vcpus": 1
1572                 },
1573                 "scheduling_parameters": {},
1574                 "secret_mounts": {},
1575                 "state": "Committed",
1576                 "use_existing": True,
1577                 'output_storage_classes': ["default"]
1578             }))
1579         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1580         mockc.open().__enter__().write.assert_has_calls([mock.call(
1581 '''{
1582   "fileblub": {
1583     "basename": "token.txt",
1584     "class": "File",
1585     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1586     "size": 0
1587   },
1588   "sleeptime": 5
1589 }''')])
1590
1591     # The test passes no builder.resources
1592     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1593     @mock.patch("arvados.collection.CollectionReader")
1594     @mock.patch("arvados.collection.Collection")
1595     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1596     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1597         arvados_cwl.add_arv_hints()
1598
1599         api = mock.MagicMock()
1600         api._rootDesc = get_rootDesc()
1601
1602         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1603         self.assertEqual(runner.work_api, 'containers')
1604
1605         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1606         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1607                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1608         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1609                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1610
1611         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1612         runner.ignore_docker_for_reuse = False
1613         runner.num_retries = 0
1614         runner.secret_store = cwltool.secrets.SecretStore()
1615
1616         loadingContext, runtimeContext = self.helper(runner)
1617         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1618         loadingContext.do_update = True
1619         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1620
1621         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1622
1623         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1624         arvtool.formatgraph = None
1625         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1626
1627         next(it).run(runtimeContext)
1628         next(it).run(runtimeContext)
1629
1630         with open("tests/wf/echo-subwf.cwl") as f:
1631             subwf = StripYAMLComments(f.read())
1632
1633         runner.api.container_requests().create.assert_called_with(
1634             body=JsonDiffMatcher({
1635                 'output_ttl': 0,
1636                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1637                 'scheduling_parameters': {},
1638                 'name': u'echo-subwf',
1639                 'secret_mounts': {},
1640                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1641                 'properties': {'cwl_input': {}},
1642                 'priority': 500,
1643                 'mounts': {
1644                     '/var/spool/cwl/cwl.input.yml': {
1645                         'portable_data_hash': '99999999999999999999999999999996+99',
1646                         'kind': 'collection',
1647                         'path': 'cwl.input.yml'
1648                     },
1649                     '/var/spool/cwl/workflow.cwl': {
1650                         'portable_data_hash': '99999999999999999999999999999996+99',
1651                         'kind': 'collection',
1652                         'path': 'workflow.cwl'
1653                     },
1654                     'stdout': {
1655                         'path': '/var/spool/cwl/cwl.output.json',
1656                         'kind': 'file'
1657                     },
1658                     '/tmp': {
1659                         'kind': 'tmp',
1660                         'capacity': 1073741824
1661                     }, '/var/spool/cwl': {
1662                         'kind': 'tmp',
1663                         'capacity': 3221225472
1664                     }
1665                 },
1666                 'state': 'Committed',
1667                 'output_path': '/var/spool/cwl',
1668                 'container_image': '99999999999999999999999999999993+99',
1669                 'command': [
1670                     u'cwltool',
1671                     u'--no-container',
1672                     u'--move-outputs',
1673                     u'--preserve-entire-environment',
1674                     u'workflow.cwl',
1675                     u'cwl.input.yml'
1676                 ],
1677                 'use_existing': True,
1678                 'output_name': u'Output from step echo-subwf',
1679                 'cwd': '/var/spool/cwl',
1680                 'output_storage_classes': ["default"]
1681             }))
1682
1683     def test_default_work_api(self):
1684         arvados_cwl.add_arv_hints()
1685
1686         api = mock.MagicMock()
1687         api._rootDesc = copy.deepcopy(get_rootDesc())
1688         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1689         self.assertEqual(runner.work_api, 'containers')