sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}}
 205
 206         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 207         runner.api.collections().get().execute.return_value = {
 208             "portable_data_hash": "99999999999999999999999999999993+99"}
 209
 210         tool = cmap({
 211             "inputs": [],
 212             "outputs": [],
 213             "hints": [{
 214                 "class": "ResourceRequirement",
 215                 "coresMin": 3,
 216                 "ramMin": 3000,
 217                 "tmpdirMin": 4000,
 218                 "outdirMin": 5000
 219             }, {
 220                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 221                 "keep_cache": 512
 222             }, {
 223                 "class": "http://arvados.org/cwl#APIRequirement",
 224             }, {
 225                 "class": "http://arvados.org/cwl#PartitionRequirement",
 226                 "partition": "blurb"
 227             }, {
 228                 "class": "http://arvados.org/cwl#IntermediateOutput",
 229                 "outputTTL": 7200
 230             }, {
 231                 "class": "WorkReuse",
 232                 "enableReuse": False
 233             }],
 234             "baseCommand": "ls",
 235             "id": "",
 236             "class": "CommandLineTool",
 237             "cwlVersion": "v1.2"
 238         })
 239
 240         loadingContext, runtimeContext = self.helper(runner)
 241         runtimeContext.name = "test_resource_requirements"
 242
 243         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 244         arvtool.formatgraph = None
 245         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 246             j.run(runtimeContext)
 247
 248         call_args, call_kwargs = runner.api.container_requests().create.call_args
 249
 250         call_body_expected = {
 251             'environment': {
 252                 'HOME': '/var/spool/cwl',
 253                 'TMPDIR': '/tmp'
 254             },
 255             'name': 'test_resource_requirements',
 256             'runtime_constraints': {
 257                 'vcpus': 3,
 258                 'ram': 3145728000,
 259                 'keep_cache_ram': 536870912,
 260                 'API': True
 261             },
 262             'use_existing': False,
 263             'priority': 500,
 264             'mounts': {
 265                 '/tmp': {'kind': 'tmp',
 266                          "capacity": 4194304000 },
 267                 '/var/spool/cwl': {'kind': 'tmp',
 268                                    "capacity": 5242880000 }
 269             },
 270             'state': 'Committed',
 271             'output_name': 'Output from step test_resource_requirements',
 272             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 273             'output_path': '/var/spool/cwl',
 274             'output_ttl': 7200,
 275             'container_image': '99999999999999999999999999999993+99',
 276             'command': ['ls'],
 277             'cwd': '/var/spool/cwl',
 278             'scheduling_parameters': {
 279                 'partitions': ['blurb']
 280             },
 281             'properties': {},
 282             'secret_mounts': {},
 283             'output_storage_classes': ["default"]
 284         }
 285
 286         call_body = call_kwargs.get('body', None)
 287         self.assertNotEqual(None, call_body)
 288         for key in call_body:
 289             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 290
 291
 292     # The test passes some fields in builder.resources
 293     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 294     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 295     @mock.patch("arvados.collection.Collection")
 296     def test_initial_work_dir(self, collection_mock, keepdocker):
 297         runner = mock.MagicMock()
 298         runner.ignore_docker_for_reuse = False
 299         runner.intermediate_output_ttl = 0
 300         runner.secret_store = cwltool.secrets.SecretStore()
 301         runner.api._rootDesc = {"revision": "20210628"}
 302
 303         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 304         runner.api.collections().get().execute.return_value = {
 305             "portable_data_hash": "99999999999999999999999999999993+99"}
 306
 307         sourcemock = mock.MagicMock()
 308         def get_collection_mock(p):
 309             if "/" in p:
 310                 return (sourcemock, p.split("/", 1)[1])
 311             else:
 312                 return (sourcemock, "")
 313         runner.fs_access.get_collection.side_effect = get_collection_mock
 314
 315         vwdmock = mock.MagicMock()
 316         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 317
 318         tool = cmap({
 319             "inputs": [],
 320             "outputs": [],
 321             "hints": [{
 322                 "class": "InitialWorkDirRequirement",
 323                 "listing": [{
 324                     "class": "File",
 325                     "basename": "foo",
 326                     "location": "keep:99999999999999999999999999999995+99/bar"
 327                 },
 328                 {
 329                     "class": "Directory",
 330                     "basename": "foo2",
 331                     "location": "keep:99999999999999999999999999999995+99"
 332                 },
 333                 {
 334                     "class": "File",
 335                     "basename": "filename",
 336                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 337                 },
 338                 {
 339                     "class": "Directory",
 340                     "basename": "subdir",
 341                     "location": "keep:99999999999999999999999999999995+99/subdir"
 342                 }                        ]
 343             }],
 344             "baseCommand": "ls",
 345             "class": "CommandLineTool",
 346             "cwlVersion": "v1.2",
 347             "id": ""
 348         })
 349
 350         loadingContext, runtimeContext = self.helper(runner)
 351         runtimeContext.name = "test_initial_work_dir"
 352
 353         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 354
 355         arvtool.formatgraph = None
 356         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 357             j.run(runtimeContext)
 358
 359         call_args, call_kwargs = runner.api.container_requests().create.call_args
 360
 361         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 365
 366         call_body_expected = {
 367             'environment': {
 368                 'HOME': '/var/spool/cwl',
 369                 'TMPDIR': '/tmp'
 370             },
 371             'name': 'test_initial_work_dir',
 372             'runtime_constraints': {
 373                 'vcpus': 1,
 374                 'ram': 268435456
 375             },
 376             'use_existing': True,
 377             'priority': 500,
 378             'mounts': {
 379                 '/tmp': {'kind': 'tmp',
 380                          "capacity": 1073741824 },
 381                 '/var/spool/cwl': {'kind': 'tmp',
 382                                    "capacity": 1073741824 },
 383                 '/var/spool/cwl/foo': {
 384                     'kind': 'collection',
 385                     'path': 'foo',
 386                     'portable_data_hash': '99999999999999999999999999999996+99'
 387                 },
 388                 '/var/spool/cwl/foo2': {
 389                     'kind': 'collection',
 390                     'path': 'foo2',
 391                     'portable_data_hash': '99999999999999999999999999999996+99'
 392                 },
 393                 '/var/spool/cwl/filename': {
 394                     'kind': 'collection',
 395                     'path': 'filename',
 396                     'portable_data_hash': '99999999999999999999999999999996+99'
 397                 },
 398                 '/var/spool/cwl/subdir': {
 399                     'kind': 'collection',
 400                     'path': 'subdir',
 401                     'portable_data_hash': '99999999999999999999999999999996+99'
 402                 }
 403             },
 404             'state': 'Committed',
 405             'output_name': 'Output from step test_initial_work_dir',
 406             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 407             'output_path': '/var/spool/cwl',
 408             'output_ttl': 0,
 409             'container_image': '99999999999999999999999999999993+99',
 410             'command': ['ls'],
 411             'cwd': '/var/spool/cwl',
 412             'scheduling_parameters': {
 413             },
 414             'properties': {},
 415             'secret_mounts': {},
 416             'output_storage_classes': ["default"]
 417         }
 418
 419         call_body = call_kwargs.get('body', None)
 420         self.assertNotEqual(None, call_body)
 421         for key in call_body:
 422             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 423
 424
 425     # Test redirecting stdin/stdout/stderr
 426     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 427     def test_redirects(self, keepdocker):
 428         runner = mock.MagicMock()
 429         runner.ignore_docker_for_reuse = False
 430         runner.intermediate_output_ttl = 0
 431         runner.secret_store = cwltool.secrets.SecretStore()
 432         runner.api._rootDesc = {"revision": "20210628"}
 433
 434         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 435         runner.api.collections().get().execute.return_value = {
 436             "portable_data_hash": "99999999999999999999999999999993+99"}
 437
 438         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 439
 440         tool = cmap({
 441             "inputs": [],
 442             "outputs": [],
 443             "baseCommand": "ls",
 444             "stdout": "stdout.txt",
 445             "stderr": "stderr.txt",
 446             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 447             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 448             "id": "",
 449             "class": "CommandLineTool",
 450             "cwlVersion": "v1.2"
 451         })
 452
 453         loadingContext, runtimeContext = self.helper(runner)
 454         runtimeContext.name = "test_run_redirect"
 455
 456         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 457         arvtool.formatgraph = None
 458         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 459             j.run(runtimeContext)
 460             runner.api.container_requests().create.assert_called_with(
 461                 body=JsonDiffMatcher({
 462                     'environment': {
 463                         'HOME': '/var/spool/cwl',
 464                         'TMPDIR': '/tmp'
 465                     },
 466                     'name': 'test_run_redirect',
 467                     'runtime_constraints': {
 468                         'vcpus': 1,
 469                         'ram': 268435456
 470                     },
 471                     'use_existing': True,
 472                     'priority': 500,
 473                     'mounts': {
 474                         '/tmp': {'kind': 'tmp',
 475                                  "capacity": 1073741824 },
 476                         '/var/spool/cwl': {'kind': 'tmp',
 477                                            "capacity": 1073741824 },
 478                         "stderr": {
 479                             "kind": "file",
 480                             "path": "/var/spool/cwl/stderr.txt"
 481                         },
 482                         "stdin": {
 483                             "kind": "collection",
 484                             "path": "file.txt",
 485                             "portable_data_hash": "99999999999999999999999999999996+99"
 486                         },
 487                         "stdout": {
 488                             "kind": "file",
 489                             "path": "/var/spool/cwl/stdout.txt"
 490                         },
 491                     },
 492                     'state': 'Committed',
 493                     "output_name": "Output from step test_run_redirect",
 494                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 495                     'output_path': '/var/spool/cwl',
 496                     'output_ttl': 0,
 497                     'container_image': '99999999999999999999999999999993+99',
 498                     'command': ['ls', '/var/spool/cwl'],
 499                     'cwd': '/var/spool/cwl',
 500                     'scheduling_parameters': {},
 501                     'properties': {},
 502                     'secret_mounts': {},
 503                     'output_storage_classes': ["default"]
 504                 }))
 505
 506     @mock.patch("arvados.collection.Collection")
 507     def test_done(self, col):
 508         api = mock.MagicMock()
 509
 510         runner = mock.MagicMock()
 511         runner.api = api
 512         runner.num_retries = 0
 513         runner.ignore_docker_for_reuse = False
 514         runner.intermediate_output_ttl = 0
 515         runner.secret_store = cwltool.secrets.SecretStore()
 516
 517         runner.api.containers().get().execute.return_value = {"state":"Complete",
 518                                                               "output": "abc+123",
 519                                                               "exit_code": 0}
 520
 521         col().open.return_value = []
 522
 523         loadingContext, runtimeContext = self.helper(runner)
 524
 525         arvjob = arvados_cwl.ArvadosContainer(runner,
 526                                               runtimeContext,
 527                                               mock.MagicMock(),
 528                                               {},
 529                                               None,
 530                                               [],
 531                                               [],
 532                                               "testjob")
 533         arvjob.output_callback = mock.MagicMock()
 534         arvjob.collect_outputs = mock.MagicMock()
 535         arvjob.successCodes = [0]
 536         arvjob.outdir = "/var/spool/cwl"
 537         arvjob.output_ttl = 3600
 538
 539         arvjob.collect_outputs.return_value = {"out": "stuff"}
 540
 541         arvjob.done({
 542             "state": "Final",
 543             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 544             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 545             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 546             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 547             "modified_at": "2017-05-26T12:01:22Z"
 548         })
 549
 550         self.assertFalse(api.collections().create.called)
 551         self.assertFalse(runner.runtime_status_error.called)
 552
 553         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 554         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 555         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 556
 557     # Test to make sure we dont call runtime_status_update if we already did
 558     # some where higher up in the call stack
 559     @mock.patch("arvados_cwl.util.get_current_container")
 560     def test_recursive_runtime_status_update(self, gcc_mock):
 561         self.setup_and_test_container_executor_and_logging(gcc_mock)
 562         root_logger = logging.getLogger('')
 563
 564         # get_current_container is invoked when we call runtime_status_update
 565         # so try and log again!
 566         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 567         try:
 568             root_logger.error("First Error")
 569         except RuntimeError:
 570             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 571
 572
 573     # Test to make sure that an exception raised from
 574     # get_current_container doesn't cause the logger to raise an
 575     # exception
 576     @mock.patch("arvados_cwl.util.get_current_container")
 577     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 578         self.setup_and_test_container_executor_and_logging(gcc_mock)
 579         root_logger = logging.getLogger('')
 580
 581         # get_current_container is invoked when we call
 582         # runtime_status_update, it is going to also raise an
 583         # exception.
 584         gcc_mock.side_effect = Exception("Second Error")
 585         try:
 586             root_logger.error("First Error")
 587         except Exception:
 588             self.fail("Exception in logger should not propagate")
 589         self.assertTrue(gcc_mock.called)
 590
 591     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 592     @mock.patch("arvados_cwl.util.get_current_container")
 593     @mock.patch("arvados.collection.CollectionReader")
 594     @mock.patch("arvados.collection.Collection")
 595     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 596         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 597
 598         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 599         self.assertTrue(gcc_mock.called)
 600
 601         runner.num_retries = 0
 602         runner.ignore_docker_for_reuse = False
 603         runner.intermediate_output_ttl = 0
 604         runner.secret_store = cwltool.secrets.SecretStore()
 605         runner.label = mock.MagicMock()
 606         runner.label.return_value = '[container testjob]'
 607
 608         runner.api.containers().get().execute.return_value = {
 609             "state":"Complete",
 610             "output": "abc+123",
 611             "exit_code": 1,
 612             "log": "def+234"
 613         }
 614
 615         col().open.return_value = []
 616
 617         loadingContext, runtimeContext = self.helper(runner)
 618
 619         arvjob = arvados_cwl.ArvadosContainer(runner,
 620                                               runtimeContext,
 621                                               mock.MagicMock(),
 622                                               {},
 623                                               None,
 624                                               [],
 625                                               [],
 626                                               "testjob")
 627         arvjob.output_callback = mock.MagicMock()
 628         arvjob.collect_outputs = mock.MagicMock()
 629         arvjob.successCodes = [0]
 630         arvjob.outdir = "/var/spool/cwl"
 631         arvjob.output_ttl = 3600
 632         arvjob.collect_outputs.return_value = {"out": "stuff"}
 633
 634         arvjob.done({
 635             "state": "Final",
 636             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 637             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 638             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 639             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 640             "modified_at": "2017-05-26T12:01:22Z"
 641         })
 642
 643         rts_mock.assert_called_with(
 644             'error',
 645             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 646             '  ** log is empty **'
 647         )
 648         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 649
 650     # The test passes no builder.resources
 651     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 652     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 653     def test_mounts(self, keepdocker):
 654         runner = mock.MagicMock()
 655         runner.ignore_docker_for_reuse = False
 656         runner.intermediate_output_ttl = 0
 657         runner.secret_store = cwltool.secrets.SecretStore()
 658         runner.api._rootDesc = {"revision": "20210628"}
 659
 660         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 661         runner.api.collections().get().execute.return_value = {
 662             "portable_data_hash": "99999999999999999999999999999994+99",
 663             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 664
 665         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 666
 667         tool = cmap({
 668             "inputs": [
 669                 {"id": "p1",
 670                  "type": "Directory"}
 671             ],
 672             "outputs": [],
 673             "baseCommand": "ls",
 674             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 675             "id": "",
 676             "class": "CommandLineTool",
 677             "cwlVersion": "v1.2"
 678         })
 679
 680         loadingContext, runtimeContext = self.helper(runner)
 681         runtimeContext.name = "test_run_mounts"
 682
 683         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 684         arvtool.formatgraph = None
 685         job_order = {
 686             "p1": {
 687                 "class": "Directory",
 688                 "location": "keep:99999999999999999999999999999994+44",
 689                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 690                 "listing": [
 691                     {
 692                         "class": "File",
 693                         "location": "keep:99999999999999999999999999999994+44/file1",
 694                     },
 695                     {
 696                         "class": "File",
 697                         "location": "keep:99999999999999999999999999999994+44/file2",
 698                     }
 699                 ]
 700             }
 701         }
 702         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 703             j.run(runtimeContext)
 704             runner.api.container_requests().create.assert_called_with(
 705                 body=JsonDiffMatcher({
 706                     'environment': {
 707                         'HOME': '/var/spool/cwl',
 708                         'TMPDIR': '/tmp'
 709                     },
 710                     'name': 'test_run_mounts',
 711                     'runtime_constraints': {
 712                         'vcpus': 1,
 713                         'ram': 268435456
 714                     },
 715                     'use_existing': True,
 716                     'priority': 500,
 717                     'mounts': {
 718                         "/keep/99999999999999999999999999999994+44": {
 719                             "kind": "collection",
 720                             "portable_data_hash": "99999999999999999999999999999994+44",
 721                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 722                         },
 723                         '/tmp': {'kind': 'tmp',
 724                                  "capacity": 1073741824 },
 725                         '/var/spool/cwl': {'kind': 'tmp',
 726                                            "capacity": 1073741824 }
 727                     },
 728                     'state': 'Committed',
 729                     'output_name': 'Output from step test_run_mounts',
 730                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 731                     'output_path': '/var/spool/cwl',
 732                     'output_ttl': 0,
 733                     'container_image': '99999999999999999999999999999994+99',
 734                     'command': ['ls', '/var/spool/cwl'],
 735                     'cwd': '/var/spool/cwl',
 736                     'scheduling_parameters': {},
 737                     'properties': {},
 738                     'secret_mounts': {},
 739                     'output_storage_classes': ["default"]
 740                 }))
 741
 742     # The test passes no builder.resources
 743     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 744     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 745     def test_secrets(self, keepdocker):
 746         arvados_cwl.add_arv_hints()
 747         runner = mock.MagicMock()
 748         runner.ignore_docker_for_reuse = False
 749         runner.intermediate_output_ttl = 0
 750         runner.secret_store = cwltool.secrets.SecretStore()
 751         runner.api._rootDesc = {"revision": "20210628"}
 752
 753         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 754         runner.api.collections().get().execute.return_value = {
 755             "portable_data_hash": "99999999999999999999999999999993+99"}
 756
 757         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 758
 759         tool = cmap({"arguments": ["md5sum", "example.conf"],
 760                      "class": "CommandLineTool",
 761                      "cwlVersion": "v1.2",
 762                      "hints": [
 763                          {
 764                              "class": "http://commonwl.org/cwltool#Secrets",
 765                              "secrets": [
 766                                  "#secret_job.cwl/pw"
 767                              ]
 768                          }
 769                      ],
 770                      "id": "",
 771                      "inputs": [
 772                          {
 773                              "id": "#secret_job.cwl/pw",
 774                              "type": "string"
 775                          }
 776                      ],
 777                      "outputs": [
 778                      ],
 779                      "requirements": [
 780                          {
 781                              "class": "InitialWorkDirRequirement",
 782                              "listing": [
 783                                  {
 784                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 785                                      "entryname": "example.conf"
 786                                  }
 787                              ]
 788                          }
 789                      ]})
 790
 791         loadingContext, runtimeContext = self.helper(runner)
 792         runtimeContext.name = "test_secrets"
 793
 794         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 795         arvtool.formatgraph = None
 796
 797         job_order = {"pw": "blorp"}
 798         runner.secret_store.store(["pw"], job_order)
 799
 800         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 801             j.run(runtimeContext)
 802             runner.api.container_requests().create.assert_called_with(
 803                 body=JsonDiffMatcher({
 804                     'environment': {
 805                         'HOME': '/var/spool/cwl',
 806                         'TMPDIR': '/tmp'
 807                     },
 808                     'name': 'test_secrets',
 809                     'runtime_constraints': {
 810                         'vcpus': 1,
 811                         'ram': 268435456
 812                     },
 813                     'use_existing': True,
 814                     'priority': 500,
 815                     'mounts': {
 816                         '/tmp': {'kind': 'tmp',
 817                                  "capacity": 1073741824
 818                              },
 819                         '/var/spool/cwl': {'kind': 'tmp',
 820                                            "capacity": 1073741824 }
 821                     },
 822                     'state': 'Committed',
 823                     'output_name': 'Output from step test_secrets',
 824                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 825                     'output_path': '/var/spool/cwl',
 826                     'output_ttl': 0,
 827                     'container_image': '99999999999999999999999999999993+99',
 828                     'command': ['md5sum', 'example.conf'],
 829                     'cwd': '/var/spool/cwl',
 830                     'scheduling_parameters': {},
 831                     'properties': {},
 832                     "secret_mounts": {
 833                         "/var/spool/cwl/example.conf": {
 834                             "content": "username: user\npassword: blorp\n",
 835                             "kind": "text"
 836                         }
 837                     },
 838                     'output_storage_classes': ["default"]
 839                 }))
 840
 841     # The test passes no builder.resources
 842     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 843     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 844     def test_timelimit(self, keepdocker):
 845         runner = mock.MagicMock()
 846         runner.ignore_docker_for_reuse = False
 847         runner.intermediate_output_ttl = 0
 848         runner.secret_store = cwltool.secrets.SecretStore()
 849         runner.api._rootDesc = {"revision": "20210628"}
 850
 851         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 852         runner.api.collections().get().execute.return_value = {
 853             "portable_data_hash": "99999999999999999999999999999993+99"}
 854
 855         tool = cmap({
 856             "inputs": [],
 857             "outputs": [],
 858             "baseCommand": "ls",
 859             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 860             "id": "",
 861             "cwlVersion": "v1.2",
 862             "class": "CommandLineTool",
 863             "hints": [
 864                 {
 865                     "class": "ToolTimeLimit",
 866                     "timelimit": 42
 867                 }
 868             ]
 869         })
 870
 871         loadingContext, runtimeContext = self.helper(runner)
 872         runtimeContext.name = "test_timelimit"
 873
 874         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 875         arvtool.formatgraph = None
 876
 877         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 878             j.run(runtimeContext)
 879
 880         _, kwargs = runner.api.container_requests().create.call_args
 881         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 882
 883
 884     # The test passes no builder.resources
 885     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 886     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 887     def test_setting_storage_class(self, keepdocker):
 888         arv_docker_clear_cache()
 889
 890         runner = mock.MagicMock()
 891         runner.ignore_docker_for_reuse = False
 892         runner.intermediate_output_ttl = 0
 893         runner.secret_store = cwltool.secrets.SecretStore()
 894         runner.api._rootDesc = {"revision": "20210628"}
 895
 896         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 897         runner.api.collections().get().execute.return_value = {
 898             "portable_data_hash": "99999999999999999999999999999993+99"}
 899
 900         tool = cmap({
 901             "inputs": [],
 902             "outputs": [],
 903             "baseCommand": "ls",
 904             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 905             "id": "",
 906             "cwlVersion": "v1.2",
 907             "class": "CommandLineTool",
 908             "hints": [
 909                 {
 910                     "class": "http://arvados.org/cwl#OutputStorageClass",
 911                     "finalStorageClass": ["baz_sc", "qux_sc"],
 912                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 913                 }
 914             ]
 915         })
 916
 917         loadingContext, runtimeContext = self.helper(runner, True)
 918
 919         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 920         arvtool.formatgraph = None
 921
 922         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 923             j.run(runtimeContext)
 924             runner.api.container_requests().create.assert_called_with(
 925                 body=JsonDiffMatcher({
 926                     'environment': {
 927                         'HOME': '/var/spool/cwl',
 928                         'TMPDIR': '/tmp'
 929                     },
 930                     'name': 'test_run_True',
 931                     'runtime_constraints': {
 932                         'vcpus': 1,
 933                         'ram': 268435456
 934                     },
 935                     'use_existing': True,
 936                     'priority': 500,
 937                     'mounts': {
 938                         '/tmp': {'kind': 'tmp',
 939                                  "capacity": 1073741824
 940                              },
 941                         '/var/spool/cwl': {'kind': 'tmp',
 942                                            "capacity": 1073741824 }
 943                     },
 944                     'state': 'Committed',
 945                     'output_name': 'Output from step test_run_True',
 946                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 947                     'output_path': '/var/spool/cwl',
 948                     'output_ttl': 0,
 949                     'container_image': '99999999999999999999999999999993+99',
 950                     'command': ['ls', '/var/spool/cwl'],
 951                     'cwd': '/var/spool/cwl',
 952                     'scheduling_parameters': {},
 953                     'properties': {},
 954                     'secret_mounts': {},
 955                     'output_storage_classes': ["foo_sc", "bar_sc"]
 956                 }))
 957
 958
 959     # The test passes no builder.resources
 960     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 961     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 962     def test_setting_process_properties(self, keepdocker):
 963         arv_docker_clear_cache()
 964
 965         runner = mock.MagicMock()
 966         runner.ignore_docker_for_reuse = False
 967         runner.intermediate_output_ttl = 0
 968         runner.secret_store = cwltool.secrets.SecretStore()
 969         runner.api._rootDesc = {"revision": "20210628"}
 970
 971         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 972         runner.api.collections().get().execute.return_value = {
 973             "portable_data_hash": "99999999999999999999999999999993+99"}
 974
 975         tool = cmap({
 976             "inputs": [
 977                 {"id": "x", "type": "string"}],
 978             "outputs": [],
 979             "baseCommand": "ls",
 980             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 981             "id": "",
 982             "class": "CommandLineTool",
 983             "cwlVersion": "v1.2",
 984             "hints": [
 985             {
 986                 "class": "http://arvados.org/cwl#ProcessProperties",
 987                 "processProperties": [
 988                     {"propertyName": "foo",
 989                      "propertyValue": "bar"},
 990                     {"propertyName": "baz",
 991                      "propertyValue": "$(inputs.x)"},
 992                     {"propertyName": "quux",
 993                      "propertyValue": {
 994                          "q1": 1,
 995                          "q2": 2
 996                      }
 997                     }
 998                 ],
 999             }
1000         ]
1001         })
1002
1003         loadingContext, runtimeContext = self.helper(runner, True)
1004
1005         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1006         arvtool.formatgraph = None
1007
1008         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1009             j.run(runtimeContext)
1010             runner.api.container_requests().create.assert_called_with(
1011                 body=JsonDiffMatcher({
1012                     'environment': {
1013                         'HOME': '/var/spool/cwl',
1014                         'TMPDIR': '/tmp'
1015                     },
1016                     'name': 'test_run_True',
1017                     'runtime_constraints': {
1018                         'vcpus': 1,
1019                         'ram': 268435456
1020                     },
1021                     'use_existing': True,
1022                     'priority': 500,
1023                     'mounts': {
1024                         '/tmp': {'kind': 'tmp',
1025                                  "capacity": 1073741824
1026                              },
1027                         '/var/spool/cwl': {'kind': 'tmp',
1028                                            "capacity": 1073741824 }
1029                     },
1030                     'state': 'Committed',
1031                     'output_name': 'Output from step test_run_True',
1032                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1033                     'output_path': '/var/spool/cwl',
1034                     'output_ttl': 0,
1035                     'container_image': '99999999999999999999999999999993+99',
1036                     'command': ['ls', '/var/spool/cwl'],
1037                     'cwd': '/var/spool/cwl',
1038                     'scheduling_parameters': {},
1039                     'properties': {
1040                         "baz": "blorp",
1041                         "foo": "bar",
1042                         "quux": {
1043                             "q1": 1,
1044                             "q2": 2
1045                         }
1046                     },
1047                     'secret_mounts': {},
1048                     'output_storage_classes': ["default"]
1049                 }))
1050
1051
1052     # The test passes no builder.resources
1053     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1054     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1055     def test_cuda_requirement(self, keepdocker):
1056         arvados_cwl.add_arv_hints()
1057         arv_docker_clear_cache()
1058
1059         runner = mock.MagicMock()
1060         runner.ignore_docker_for_reuse = False
1061         runner.intermediate_output_ttl = 0
1062         runner.secret_store = cwltool.secrets.SecretStore()
1063         runner.api._rootDesc = {"revision": "20210628"}
1064
1065         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1066         runner.api.collections().get().execute.return_value = {
1067             "portable_data_hash": "99999999999999999999999999999993+99"}
1068
1069         test_cwl_req = [{
1070                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1071                 "cudaVersionMin": "11.0",
1072                 "cudaComputeCapability": "9.0",
1073             }, {
1074                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1075                 "cudaVersionMin": "11.0",
1076                 "cudaComputeCapability": "9.0",
1077                 "cudaDeviceCountMin": 2
1078             }, {
1079                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1080                 "cudaVersionMin": "11.0",
1081                 "cudaComputeCapability": ["4.0", "5.0"],
1082                 "cudaDeviceCountMin": 2
1083             }]
1084
1085         test_arv_req = [{
1086             'device_count': 1,
1087             'driver_version': "11.0",
1088             'hardware_capability': "9.0"
1089         }, {
1090             'device_count': 2,
1091             'driver_version': "11.0",
1092             'hardware_capability': "9.0"
1093         }, {
1094             'device_count': 2,
1095             'driver_version': "11.0",
1096             'hardware_capability': "4.0"
1097         }]
1098
1099         for test_case in range(0, len(test_cwl_req)):
1100
1101             tool = cmap({
1102                 "inputs": [],
1103                 "outputs": [],
1104                 "baseCommand": "nvidia-smi",
1105                 "arguments": [],
1106                 "id": "",
1107                 "cwlVersion": "v1.2",
1108                 "class": "CommandLineTool",
1109                 "requirements": [test_cwl_req[test_case]]
1110             })
1111
1112             loadingContext, runtimeContext = self.helper(runner, True)
1113
1114             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1115             arvtool.formatgraph = None
1116
1117             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1118                 j.run(runtimeContext)
1119                 runner.api.container_requests().create.assert_called_with(
1120                     body=JsonDiffMatcher({
1121                         'environment': {
1122                             'HOME': '/var/spool/cwl',
1123                             'TMPDIR': '/tmp'
1124                         },
1125                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1126                         'runtime_constraints': {
1127                             'vcpus': 1,
1128                             'ram': 268435456,
1129                             'cuda': test_arv_req[test_case]
1130                         },
1131                         'use_existing': True,
1132                         'priority': 500,
1133                         'mounts': {
1134                             '/tmp': {'kind': 'tmp',
1135                                      "capacity": 1073741824
1136                                  },
1137                             '/var/spool/cwl': {'kind': 'tmp',
1138                                                "capacity": 1073741824 }
1139                         },
1140                         'state': 'Committed',
1141                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1142                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1143                         'output_path': '/var/spool/cwl',
1144                         'output_ttl': 0,
1145                         'container_image': '99999999999999999999999999999993+99',
1146                         'command': ['nvidia-smi'],
1147                         'cwd': '/var/spool/cwl',
1148                         'scheduling_parameters': {},
1149                         'properties': {},
1150                         'secret_mounts': {},
1151                         'output_storage_classes': ["default"]
1152                     }))
1153
1154
1155     # The test passes no builder.resources
1156     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1157     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1158     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1159     def test_match_local_docker(self, keepdocker, determine_image_id):
1160         arvados_cwl.add_arv_hints()
1161         arv_docker_clear_cache()
1162
1163         runner = mock.MagicMock()
1164         runner.ignore_docker_for_reuse = False
1165         runner.intermediate_output_ttl = 0
1166         runner.secret_store = cwltool.secrets.SecretStore()
1167         runner.api._rootDesc = {"revision": "20210628"}
1168
1169         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1170                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1171         determine_image_id.side_effect = lambda x: "123"
1172         def execute(uuid):
1173             ex = mock.MagicMock()
1174             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1175                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1176             ex.execute.return_value = lookup[uuid]
1177             return ex
1178         runner.api.collections().get.side_effect = execute
1179
1180         tool = cmap({
1181             "inputs": [],
1182             "outputs": [],
1183             "baseCommand": "echo",
1184             "arguments": [],
1185             "id": "",
1186             "cwlVersion": "v1.0",
1187             "class": "org.w3id.cwl.cwl.CommandLineTool"
1188         })
1189
1190         loadingContext, runtimeContext = self.helper(runner, True)
1191
1192         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1193         arvtool.formatgraph = None
1194
1195         container_request = {
1196             'environment': {
1197                 'HOME': '/var/spool/cwl',
1198                 'TMPDIR': '/tmp'
1199             },
1200             'name': 'test_run_True',
1201             'runtime_constraints': {
1202                 'vcpus': 1,
1203                 'ram': 1073741824,
1204             },
1205             'use_existing': True,
1206             'priority': 500,
1207             'mounts': {
1208                 '/tmp': {'kind': 'tmp',
1209                          "capacity": 1073741824
1210                          },
1211                 '/var/spool/cwl': {'kind': 'tmp',
1212                                    "capacity": 1073741824 }
1213             },
1214             'state': 'Committed',
1215             'output_name': 'Output from step test_run_True',
1216             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1217             'output_path': '/var/spool/cwl',
1218             'output_ttl': 0,
1219             'container_image': '99999999999999999999999999999994+99',
1220             'command': ['echo'],
1221             'cwd': '/var/spool/cwl',
1222             'scheduling_parameters': {},
1223             'properties': {},
1224             'secret_mounts': {},
1225             'output_storage_classes': ["default"]
1226         }
1227
1228         runtimeContext.match_local_docker = False
1229         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1230             j.run(runtimeContext)
1231             runner.api.container_requests().create.assert_called_with(
1232                 body=JsonDiffMatcher(container_request))
1233
1234         arv_docker_clear_cache()
1235         runtimeContext.match_local_docker = True
1236         container_request['container_image'] = '99999999999999999999999999999993+99'
1237         container_request['name'] = 'test_run_True_2'
1238         container_request['output_name'] = 'Output from step test_run_True_2'
1239         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1240             j.run(runtimeContext)
1241             runner.api.container_requests().create.assert_called_with(
1242                 body=JsonDiffMatcher(container_request))
1243
1244
1245     # The test passes no builder.resources
1246     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1247     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1248     def test_run_preemptible_hint(self, keepdocker):
1249         arvados_cwl.add_arv_hints()
1250         for enable_preemptible in (None, True, False):
1251             for preemptible_hint in (None, True, False):
1252                 arv_docker_clear_cache()
1253
1254                 runner = mock.MagicMock()
1255                 runner.ignore_docker_for_reuse = False
1256                 runner.intermediate_output_ttl = 0
1257                 runner.secret_store = cwltool.secrets.SecretStore()
1258                 runner.api._rootDesc = {"revision": "20210628"}
1259
1260                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1261                 runner.api.collections().get().execute.return_value = {
1262                     "portable_data_hash": "99999999999999999999999999999993+99"}
1263
1264                 if preemptible_hint is not None:
1265                     hints = [{
1266                         "class": "http://arvados.org/cwl#UsePreemptible",
1267                         "usePreemptible": preemptible_hint
1268                     }]
1269                 else:
1270                     hints = []
1271
1272                 tool = cmap({
1273                     "inputs": [],
1274                     "outputs": [],
1275                     "baseCommand": "ls",
1276                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1277                     "id": "",
1278                     "class": "CommandLineTool",
1279                     "cwlVersion": "v1.2",
1280                     "hints": hints
1281                 })
1282
1283                 loadingContext, runtimeContext = self.helper(runner)
1284
1285                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1286                 runtimeContext.enable_preemptible = enable_preemptible
1287
1288                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1289                 arvtool.formatgraph = None
1290
1291                 # Test the interactions between --enable/disable-preemptible
1292                 # and UsePreemptible hint
1293
1294                 if enable_preemptible is None:
1295                     if preemptible_hint is None:
1296                         sched = {}
1297                     else:
1298                         sched = {'preemptible': preemptible_hint}
1299                 else:
1300                     if preemptible_hint is None:
1301                         sched = {'preemptible': enable_preemptible}
1302                     else:
1303                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1304
1305                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1306                     j.run(runtimeContext)
1307                     runner.api.container_requests().create.assert_called_with(
1308                         body=JsonDiffMatcher({
1309                             'environment': {
1310                                 'HOME': '/var/spool/cwl',
1311                                 'TMPDIR': '/tmp'
1312                             },
1313                             'name': runtimeContext.name,
1314                             'runtime_constraints': {
1315                                 'vcpus': 1,
1316                                 'ram': 268435456
1317                             },
1318                             'use_existing': True,
1319                             'priority': 500,
1320                             'mounts': {
1321                                 '/tmp': {'kind': 'tmp',
1322                                          "capacity": 1073741824
1323                                      },
1324                                 '/var/spool/cwl': {'kind': 'tmp',
1325                                                    "capacity": 1073741824 }
1326                             },
1327                             'state': 'Committed',
1328                             'output_name': 'Output from step '+runtimeContext.name,
1329                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1330                             'output_path': '/var/spool/cwl',
1331                             'output_ttl': 0,
1332                             'container_image': '99999999999999999999999999999993+99',
1333                             'command': ['ls', '/var/spool/cwl'],
1334                             'cwd': '/var/spool/cwl',
1335                             'scheduling_parameters': sched,
1336                             'properties': {},
1337                             'secret_mounts': {},
1338                             'output_storage_classes': ["default"]
1339                         }))
1340
1341
1342     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1343     def test_output_properties(self, keepdocker):
1344         arvados_cwl.add_arv_hints()
1345         for rev in ["20210628", "20220510"]:
1346             runner = mock.MagicMock()
1347             runner.ignore_docker_for_reuse = False
1348             runner.intermediate_output_ttl = 0
1349             runner.secret_store = cwltool.secrets.SecretStore()
1350             runner.api._rootDesc = {"revision": rev}
1351
1352             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1353             runner.api.collections().get().execute.return_value = {
1354                 "portable_data_hash": "99999999999999999999999999999993+99"}
1355
1356             tool = cmap({
1357                 "inputs": [{
1358                     "id": "inp",
1359                     "type": "string"
1360                 }],
1361                 "outputs": [],
1362                 "baseCommand": "ls",
1363                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1364                 "id": "",
1365                 "cwlVersion": "v1.2",
1366                 "class": "CommandLineTool",
1367                 "hints": [
1368                     {
1369                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1370                         "outputProperties": {
1371                             "foo": "bar",
1372                             "baz": "$(inputs.inp)"
1373                         }
1374                     }
1375                 ]
1376             })
1377
1378             loadingContext, runtimeContext = self.helper(runner)
1379             runtimeContext.name = "test_timelimit"
1380
1381             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1382             arvtool.formatgraph = None
1383
1384             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1385                 j.run(runtimeContext)
1386
1387             _, kwargs = runner.api.container_requests().create.call_args
1388             if rev == "20220510":
1389                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1390             else:
1391                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1392
1393
1394 class TestWorkflow(unittest.TestCase):
1395     def setUp(self):
1396         cwltool.process._names = set()
1397         arv_docker_clear_cache()
1398
1399     def helper(self, runner, enable_reuse=True):
1400         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1401
1402         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1403                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1404
1405         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1406         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1407         document_loader.fetch_text = document_loader.fetcher.fetch_text
1408         document_loader.check_exists = document_loader.fetcher.check_exists
1409
1410         loadingContext = arvados_cwl.context.ArvLoadingContext(
1411             {"avsc_names": avsc_names,
1412              "basedir": "",
1413              "make_fs_access": make_fs_access,
1414              "loader": document_loader,
1415              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1416              "construct_tool_object": runner.arv_make_tool})
1417         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1418             {"work_api": "containers",
1419              "basedir": "",
1420              "name": "test_run_wf_"+str(enable_reuse),
1421              "make_fs_access": make_fs_access,
1422              "tmpdir": "/tmp",
1423              "enable_reuse": enable_reuse,
1424              "priority": 500})
1425
1426         return loadingContext, runtimeContext
1427
1428     # The test passes no builder.resources
1429     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1430     @mock.patch("arvados.collection.CollectionReader")
1431     @mock.patch("arvados.collection.Collection")
1432     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1433     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1434         arvados_cwl.add_arv_hints()
1435
1436         api = mock.MagicMock()
1437         api._rootDesc = get_rootDesc()
1438
1439         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1440         self.assertEqual(runner.work_api, 'containers')
1441
1442         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1443         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1444         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1445                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1446
1447         runner.api.containers().current().execute.return_value = {}
1448
1449         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1450         runner.ignore_docker_for_reuse = False
1451         runner.num_retries = 0
1452         runner.secret_store = cwltool.secrets.SecretStore()
1453
1454         loadingContext, runtimeContext = self.helper(runner)
1455         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1456
1457         mockcollectionreader().exists.return_value = True
1458
1459         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1460         metadata["cwlVersion"] = tool["cwlVersion"]
1461
1462         mockc = mock.MagicMock()
1463         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1464         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1465
1466         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1467         arvtool.formatgraph = None
1468         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1469
1470         next(it).run(runtimeContext)
1471         next(it).run(runtimeContext)
1472
1473         with open("tests/wf/scatter2_subwf.cwl") as f:
1474             subwf = StripYAMLComments(f.read()).rstrip()
1475
1476         runner.api.container_requests().create.assert_called_with(
1477             body=JsonDiffMatcher({
1478                 "command": [
1479                     "cwltool",
1480                     "--no-container",
1481                     "--move-outputs",
1482                     "--preserve-entire-environment",
1483                     "workflow.cwl",
1484                     "cwl.input.yml"
1485                 ],
1486                 "container_image": "99999999999999999999999999999993+99",
1487                 "cwd": "/var/spool/cwl",
1488                 "environment": {
1489                     "HOME": "/var/spool/cwl",
1490                     "TMPDIR": "/tmp"
1491                 },
1492                 "mounts": {
1493                     "/keep/99999999999999999999999999999999+118": {
1494                         "kind": "collection",
1495                         "portable_data_hash": "99999999999999999999999999999999+118"
1496                     },
1497                     "/tmp": {
1498                         "capacity": 1073741824,
1499                         "kind": "tmp"
1500                     },
1501                     "/var/spool/cwl": {
1502                         "capacity": 1073741824,
1503                         "kind": "tmp"
1504                     },
1505                     "/var/spool/cwl/cwl.input.yml": {
1506                         "kind": "collection",
1507                         "path": "cwl.input.yml",
1508                         "portable_data_hash": "99999999999999999999999999999996+99"
1509                     },
1510                     "/var/spool/cwl/workflow.cwl": {
1511                         "kind": "collection",
1512                         "path": "workflow.cwl",
1513                         "portable_data_hash": "99999999999999999999999999999996+99"
1514                     },
1515                     "stdout": {
1516                         "kind": "file",
1517                         "path": "/var/spool/cwl/cwl.output.json"
1518                     }
1519                 },
1520                 "name": "scatterstep",
1521                 "output_name": "Output from step scatterstep",
1522                 "output_path": "/var/spool/cwl",
1523                 "output_ttl": 0,
1524                 "priority": 500,
1525                 "properties": {},
1526                 "runtime_constraints": {
1527                     "ram": 1073741824,
1528                     "vcpus": 1
1529                 },
1530                 "scheduling_parameters": {},
1531                 "secret_mounts": {},
1532                 "state": "Committed",
1533                 "use_existing": True,
1534                 'output_storage_classes': ["default"]
1535             }))
1536         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1537         mockc.open().__enter__().write.assert_has_calls([mock.call(
1538 '''{
1539   "fileblub": {
1540     "basename": "token.txt",
1541     "class": "File",
1542     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1543     "size": 0
1544   },
1545   "sleeptime": 5
1546 }''')])
1547
1548     # The test passes no builder.resources
1549     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1550     @mock.patch("arvados.collection.CollectionReader")
1551     @mock.patch("arvados.collection.Collection")
1552     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1553     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1554         arvados_cwl.add_arv_hints()
1555
1556         api = mock.MagicMock()
1557         api._rootDesc = get_rootDesc()
1558
1559         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1560         self.assertEqual(runner.work_api, 'containers')
1561
1562         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1563         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1564                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1565         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1566                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1567
1568         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1569         runner.ignore_docker_for_reuse = False
1570         runner.num_retries = 0
1571         runner.secret_store = cwltool.secrets.SecretStore()
1572
1573         loadingContext, runtimeContext = self.helper(runner)
1574         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1575         loadingContext.do_update = True
1576         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1577
1578         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1579
1580         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1581         arvtool.formatgraph = None
1582         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1583
1584         next(it).run(runtimeContext)
1585         next(it).run(runtimeContext)
1586
1587         with open("tests/wf/echo-subwf.cwl") as f:
1588             subwf = StripYAMLComments(f.read())
1589
1590         runner.api.container_requests().create.assert_called_with(
1591             body=JsonDiffMatcher({
1592                 'output_ttl': 0,
1593                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1594                 'scheduling_parameters': {},
1595                 'name': u'echo-subwf',
1596                 'secret_mounts': {},
1597                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1598                 'properties': {},
1599                 'priority': 500,
1600                 'mounts': {
1601                     '/var/spool/cwl/cwl.input.yml': {
1602                         'portable_data_hash': '99999999999999999999999999999996+99',
1603                         'kind': 'collection',
1604                         'path': 'cwl.input.yml'
1605                     },
1606                     '/var/spool/cwl/workflow.cwl': {
1607                         'portable_data_hash': '99999999999999999999999999999996+99',
1608                         'kind': 'collection',
1609                         'path': 'workflow.cwl'
1610                     },
1611                     'stdout': {
1612                         'path': '/var/spool/cwl/cwl.output.json',
1613                         'kind': 'file'
1614                     },
1615                     '/tmp': {
1616                         'kind': 'tmp',
1617                         'capacity': 1073741824
1618                     }, '/var/spool/cwl': {
1619                         'kind': 'tmp',
1620                         'capacity': 3221225472
1621                     }
1622                 },
1623                 'state': 'Committed',
1624                 'output_path': '/var/spool/cwl',
1625                 'container_image': '99999999999999999999999999999993+99',
1626                 'command': [
1627                     u'cwltool',
1628                     u'--no-container',
1629                     u'--move-outputs',
1630                     u'--preserve-entire-environment',
1631                     u'workflow.cwl',
1632                     u'cwl.input.yml'
1633                 ],
1634                 'use_existing': True,
1635                 'output_name': u'Output from step echo-subwf',
1636                 'cwd': '/var/spool/cwl',
1637                 'output_storage_classes': ["default"]
1638             }))
1639
1640     def test_default_work_api(self):
1641         arvados_cwl.add_arv_hints()
1642
1643         api = mock.MagicMock()
1644         api._rootDesc = copy.deepcopy(get_rootDesc())
1645         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1646         self.assertEqual(runner.work_api, 'containers')