sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {'cwl_input': {}},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         tool = cmap({
 210             "inputs": [],
 211             "outputs": [],
 212             "hints": [{
 213                 "class": "ResourceRequirement",
 214                 "coresMin": 3,
 215                 "ramMin": 3000,
 216                 "tmpdirMin": 4000,
 217                 "outdirMin": 5000
 218             }, {
 219                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 220                 "keep_cache": 512
 221             }, {
 222                 "class": "http://arvados.org/cwl#APIRequirement",
 223             }, {
 224                 "class": "http://arvados.org/cwl#PartitionRequirement",
 225                 "partition": "blurb"
 226             }, {
 227                 "class": "http://arvados.org/cwl#IntermediateOutput",
 228                 "outputTTL": 7200
 229             }, {
 230                 "class": "WorkReuse",
 231                 "enableReuse": False
 232             }],
 233             "baseCommand": "ls",
 234             "id": "",
 235             "class": "CommandLineTool",
 236             "cwlVersion": "v1.2"
 237         })
 238
 239         loadingContext, runtimeContext = self.helper(runner)
 240         runtimeContext.name = "test_resource_requirements"
 241
 242         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 243         arvtool.formatgraph = None
 244         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 245             j.run(runtimeContext)
 246
 247         call_args, call_kwargs = runner.api.container_requests().create.call_args
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_resource_requirements',
 255             'runtime_constraints': {
 256                 'vcpus': 3,
 257                 'ram': 3145728000,
 258                 'keep_cache_ram': 536870912,
 259                 'API': True
 260             },
 261             'use_existing': False,
 262             'priority': 500,
 263             'mounts': {
 264                 '/tmp': {'kind': 'tmp',
 265                          "capacity": 4194304000 },
 266                 '/var/spool/cwl': {'kind': 'tmp',
 267                                    "capacity": 5242880000 }
 268             },
 269             'state': 'Committed',
 270             'output_name': 'Output from step test_resource_requirements',
 271             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272             'output_path': '/var/spool/cwl',
 273             'output_ttl': 7200,
 274             'container_image': '99999999999999999999999999999993+99',
 275             'command': ['ls'],
 276             'cwd': '/var/spool/cwl',
 277             'scheduling_parameters': {
 278                 'partitions': ['blurb']
 279             },
 280             'properties': {'cwl_input': {}},
 281             'secret_mounts': {},
 282             'output_storage_classes': ["default"]
 283         }
 284
 285         call_body = call_kwargs.get('body', None)
 286         self.assertNotEqual(None, call_body)
 287         for key in call_body:
 288             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 289
 290
 291     # The test passes some fields in builder.resources
 292     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 293     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 294     @mock.patch("arvados.collection.Collection")
 295     def test_initial_work_dir(self, collection_mock, keepdocker):
 296         runner = mock.MagicMock()
 297         runner.ignore_docker_for_reuse = False
 298         runner.intermediate_output_ttl = 0
 299         runner.secret_store = cwltool.secrets.SecretStore()
 300         runner.api._rootDesc = {"revision": "20210628"}
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         sourcemock = mock.MagicMock()
 307         def get_collection_mock(p):
 308             if "/" in p:
 309                 return (sourcemock, p.split("/", 1)[1])
 310             else:
 311                 return (sourcemock, "")
 312         runner.fs_access.get_collection.side_effect = get_collection_mock
 313
 314         vwdmock = mock.MagicMock()
 315         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 316
 317         tool = cmap({
 318             "inputs": [],
 319             "outputs": [],
 320             "hints": [{
 321                 "class": "InitialWorkDirRequirement",
 322                 "listing": [{
 323                     "class": "File",
 324                     "basename": "foo",
 325                     "location": "keep:99999999999999999999999999999995+99/bar"
 326                 },
 327                 {
 328                     "class": "Directory",
 329                     "basename": "foo2",
 330                     "location": "keep:99999999999999999999999999999995+99"
 331                 },
 332                 {
 333                     "class": "File",
 334                     "basename": "filename",
 335                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 336                 },
 337                 {
 338                     "class": "Directory",
 339                     "basename": "subdir",
 340                     "location": "keep:99999999999999999999999999999995+99/subdir"
 341                 }                        ]
 342             }],
 343             "baseCommand": "ls",
 344             "class": "CommandLineTool",
 345             "cwlVersion": "v1.2",
 346             "id": ""
 347         })
 348
 349         loadingContext, runtimeContext = self.helper(runner)
 350         runtimeContext.name = "test_initial_work_dir"
 351
 352         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 353
 354         arvtool.formatgraph = None
 355         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 356             j.run(runtimeContext)
 357
 358         call_args, call_kwargs = runner.api.container_requests().create.call_args
 359
 360         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 361         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 364
 365         call_body_expected = {
 366             'environment': {
 367                 'HOME': '/var/spool/cwl',
 368                 'TMPDIR': '/tmp'
 369             },
 370             'name': 'test_initial_work_dir',
 371             'runtime_constraints': {
 372                 'vcpus': 1,
 373                 'ram': 268435456
 374             },
 375             'use_existing': True,
 376             'priority': 500,
 377             'mounts': {
 378                 '/tmp': {'kind': 'tmp',
 379                          "capacity": 1073741824 },
 380                 '/var/spool/cwl': {'kind': 'tmp',
 381                                    "capacity": 1073741824 },
 382                 '/var/spool/cwl/foo': {
 383                     'kind': 'collection',
 384                     'path': 'foo',
 385                     'portable_data_hash': '99999999999999999999999999999996+99'
 386                 },
 387                 '/var/spool/cwl/foo2': {
 388                     'kind': 'collection',
 389                     'path': 'foo2',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/filename': {
 393                     'kind': 'collection',
 394                     'path': 'filename',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/subdir': {
 398                     'kind': 'collection',
 399                     'path': 'subdir',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 }
 402             },
 403             'state': 'Committed',
 404             'output_name': 'Output from step test_initial_work_dir',
 405             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 406             'output_path': '/var/spool/cwl',
 407             'output_ttl': 0,
 408             'container_image': '99999999999999999999999999999993+99',
 409             'command': ['ls'],
 410             'cwd': '/var/spool/cwl',
 411             'scheduling_parameters': {
 412             },
 413             'properties': {'cwl_input': {}},
 414             'secret_mounts': {},
 415             'output_storage_classes': ["default"]
 416         }
 417
 418         call_body = call_kwargs.get('body', None)
 419         self.assertNotEqual(None, call_body)
 420         for key in call_body:
 421             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 422
 423
 424     # Test redirecting stdin/stdout/stderr
 425     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 426     def test_redirects(self, keepdocker):
 427         runner = mock.MagicMock()
 428         runner.ignore_docker_for_reuse = False
 429         runner.intermediate_output_ttl = 0
 430         runner.secret_store = cwltool.secrets.SecretStore()
 431         runner.api._rootDesc = {"revision": "20210628"}
 432
 433         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 434         runner.api.collections().get().execute.return_value = {
 435             "portable_data_hash": "99999999999999999999999999999993+99"}
 436
 437         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 438
 439         tool = cmap({
 440             "inputs": [],
 441             "outputs": [],
 442             "baseCommand": "ls",
 443             "stdout": "stdout.txt",
 444             "stderr": "stderr.txt",
 445             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 446             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 447             "id": "",
 448             "class": "CommandLineTool",
 449             "cwlVersion": "v1.2"
 450         })
 451
 452         loadingContext, runtimeContext = self.helper(runner)
 453         runtimeContext.name = "test_run_redirect"
 454
 455         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 456         arvtool.formatgraph = None
 457         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 458             j.run(runtimeContext)
 459             runner.api.container_requests().create.assert_called_with(
 460                 body=JsonDiffMatcher({
 461                     'environment': {
 462                         'HOME': '/var/spool/cwl',
 463                         'TMPDIR': '/tmp'
 464                     },
 465                     'name': 'test_run_redirect',
 466                     'runtime_constraints': {
 467                         'vcpus': 1,
 468                         'ram': 268435456
 469                     },
 470                     'use_existing': True,
 471                     'priority': 500,
 472                     'mounts': {
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 },
 477                         "stderr": {
 478                             "kind": "file",
 479                             "path": "/var/spool/cwl/stderr.txt"
 480                         },
 481                         "stdin": {
 482                             "kind": "collection",
 483                             "path": "file.txt",
 484                             "portable_data_hash": "99999999999999999999999999999996+99"
 485                         },
 486                         "stdout": {
 487                             "kind": "file",
 488                             "path": "/var/spool/cwl/stdout.txt"
 489                         },
 490                     },
 491                     'state': 'Committed',
 492                     "output_name": "Output from step test_run_redirect",
 493                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 494                     'output_path': '/var/spool/cwl',
 495                     'output_ttl': 0,
 496                     'container_image': '99999999999999999999999999999993+99',
 497                     'command': ['ls', '/var/spool/cwl'],
 498                     'cwd': '/var/spool/cwl',
 499                     'scheduling_parameters': {},
 500                     'properties': {'cwl_input': {}},
 501                     'secret_mounts': {},
 502                     'output_storage_classes': ["default"]
 503                 }))
 504
 505     @mock.patch("arvados.collection.Collection")
 506     def test_done(self, col):
 507         api = mock.MagicMock()
 508
 509         runner = mock.MagicMock()
 510         runner.api = api
 511         runner.num_retries = 0
 512         runner.ignore_docker_for_reuse = False
 513         runner.intermediate_output_ttl = 0
 514         runner.secret_store = cwltool.secrets.SecretStore()
 515
 516         runner.api.containers().get().execute.return_value = {"state":"Complete",
 517                                                               "output": "abc+123",
 518                                                               "exit_code": 0}
 519
 520         col().open.return_value = []
 521
 522         loadingContext, runtimeContext = self.helper(runner)
 523
 524         arvjob = arvados_cwl.ArvadosContainer(runner,
 525                                               runtimeContext,
 526                                               mock.MagicMock(),
 527                                               {},
 528                                               None,
 529                                               [],
 530                                               [],
 531                                               "testjob")
 532         arvjob.output_callback = mock.MagicMock()
 533         arvjob.collect_outputs = mock.MagicMock()
 534         arvjob.successCodes = [0]
 535         arvjob.outdir = "/var/spool/cwl"
 536         arvjob.output_ttl = 3600
 537
 538         arvjob.collect_outputs.return_value = {"out": "stuff"}
 539
 540         arvjob.done({
 541             "state": "Final",
 542             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 543             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 544             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 545             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 546             "modified_at": "2017-05-26T12:01:22Z",
 547             "properties": {}
 548         })
 549
 550         self.assertFalse(api.collections().create.called)
 551         self.assertFalse(runner.runtime_status_error.called)
 552
 553         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 554         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 555         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 556
 557     # Test to make sure we dont call runtime_status_update if we already did
 558     # some where higher up in the call stack
 559     @mock.patch("arvados_cwl.util.get_current_container")
 560     def test_recursive_runtime_status_update(self, gcc_mock):
 561         self.setup_and_test_container_executor_and_logging(gcc_mock)
 562         root_logger = logging.getLogger('')
 563
 564         # get_current_container is invoked when we call runtime_status_update
 565         # so try and log again!
 566         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 567         try:
 568             root_logger.error("First Error")
 569         except RuntimeError:
 570             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 571
 572
 573     # Test to make sure that an exception raised from
 574     # get_current_container doesn't cause the logger to raise an
 575     # exception
 576     @mock.patch("arvados_cwl.util.get_current_container")
 577     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 578         self.setup_and_test_container_executor_and_logging(gcc_mock)
 579         root_logger = logging.getLogger('')
 580
 581         # get_current_container is invoked when we call
 582         # runtime_status_update, it is going to also raise an
 583         # exception.
 584         gcc_mock.side_effect = Exception("Second Error")
 585         try:
 586             root_logger.error("First Error")
 587         except Exception:
 588             self.fail("Exception in logger should not propagate")
 589         self.assertTrue(gcc_mock.called)
 590
 591     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 592     @mock.patch("arvados_cwl.util.get_current_container")
 593     @mock.patch("arvados.collection.CollectionReader")
 594     @mock.patch("arvados.collection.Collection")
 595     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 596         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 597
 598         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 599         self.assertTrue(gcc_mock.called)
 600
 601         runner.num_retries = 0
 602         runner.ignore_docker_for_reuse = False
 603         runner.intermediate_output_ttl = 0
 604         runner.secret_store = cwltool.secrets.SecretStore()
 605         runner.label = mock.MagicMock()
 606         runner.label.return_value = '[container testjob]'
 607
 608         runner.api.containers().get().execute.return_value = {
 609             "state":"Complete",
 610             "output": "abc+123",
 611             "exit_code": 1,
 612             "log": "def+234"
 613         }
 614
 615         col().open.return_value = []
 616
 617         loadingContext, runtimeContext = self.helper(runner)
 618
 619         arvjob = arvados_cwl.ArvadosContainer(runner,
 620                                               runtimeContext,
 621                                               mock.MagicMock(),
 622                                               {},
 623                                               None,
 624                                               [],
 625                                               [],
 626                                               "testjob")
 627         arvjob.output_callback = mock.MagicMock()
 628         arvjob.collect_outputs = mock.MagicMock()
 629         arvjob.successCodes = [0]
 630         arvjob.outdir = "/var/spool/cwl"
 631         arvjob.output_ttl = 3600
 632         arvjob.collect_outputs.return_value = {"out": "stuff"}
 633
 634         arvjob.done({
 635             "state": "Final",
 636             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 637             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 638             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 639             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 640             "modified_at": "2017-05-26T12:01:22Z",
 641             "properties": {}
 642         })
 643
 644         rts_mock.assert_called_with(
 645             'error',
 646             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 647             '  ** log is empty **'
 648         )
 649         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 650
 651     # The test passes no builder.resources
 652     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 653     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 654     def test_mounts(self, keepdocker):
 655         runner = mock.MagicMock()
 656         runner.ignore_docker_for_reuse = False
 657         runner.intermediate_output_ttl = 0
 658         runner.secret_store = cwltool.secrets.SecretStore()
 659         runner.api._rootDesc = {"revision": "20210628"}
 660
 661         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 662         runner.api.collections().get().execute.return_value = {
 663             "portable_data_hash": "99999999999999999999999999999994+99",
 664             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 665
 666         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 667
 668         tool = cmap({
 669             "inputs": [
 670                 {"id": "p1",
 671                  "type": "Directory"}
 672             ],
 673             "outputs": [],
 674             "baseCommand": "ls",
 675             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 676             "id": "",
 677             "class": "CommandLineTool",
 678             "cwlVersion": "v1.2"
 679         })
 680
 681         loadingContext, runtimeContext = self.helper(runner)
 682         runtimeContext.name = "test_run_mounts"
 683
 684         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 685         arvtool.formatgraph = None
 686         job_order = {
 687             "p1": {
 688                 "class": "Directory",
 689                 "location": "keep:99999999999999999999999999999994+44",
 690                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 691                 "listing": [
 692                     {
 693                         "class": "File",
 694                         "location": "keep:99999999999999999999999999999994+44/file1",
 695                     },
 696                     {
 697                         "class": "File",
 698                         "location": "keep:99999999999999999999999999999994+44/file2",
 699                     }
 700                 ]
 701             }
 702         }
 703         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 704             j.run(runtimeContext)
 705             runner.api.container_requests().create.assert_called_with(
 706                 body=JsonDiffMatcher({
 707                     'environment': {
 708                         'HOME': '/var/spool/cwl',
 709                         'TMPDIR': '/tmp'
 710                     },
 711                     'name': 'test_run_mounts',
 712                     'runtime_constraints': {
 713                         'vcpus': 1,
 714                         'ram': 268435456
 715                     },
 716                     'use_existing': True,
 717                     'priority': 500,
 718                     'mounts': {
 719                         "/keep/99999999999999999999999999999994+44": {
 720                             "kind": "collection",
 721                             "portable_data_hash": "99999999999999999999999999999994+44",
 722                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 723                         },
 724                         '/tmp': {'kind': 'tmp',
 725                                  "capacity": 1073741824 },
 726                         '/var/spool/cwl': {'kind': 'tmp',
 727                                            "capacity": 1073741824 }
 728                     },
 729                     'state': 'Committed',
 730                     'output_name': 'Output from step test_run_mounts',
 731                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 732                     'output_path': '/var/spool/cwl',
 733                     'output_ttl': 0,
 734                     'container_image': '99999999999999999999999999999994+99',
 735                     'command': ['ls', '/var/spool/cwl'],
 736                     'cwd': '/var/spool/cwl',
 737                     'scheduling_parameters': {},
 738                     'properties': {'cwl_input': {
 739                         "p1": {
 740                             "basename": "99999999999999999999999999999994+44",
 741                             "class": "Directory",
 742                             "dirname": "/keep",
 743                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 744                             "listing": [
 745                                 {
 746                                     "basename": "file1",
 747                                     "class": "File",
 748                                     "dirname": "/keep/99999999999999999999999999999994+44",
 749                                     "location": "keep:99999999999999999999999999999994+44/file1",
 750                                     "nameext": "",
 751                                     "nameroot": "file1",
 752                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 753                                     "size": 0
 754                                 },
 755                                 {
 756                                     "basename": "file2",
 757                                     "class": "File",
 758                                     "dirname": "/keep/99999999999999999999999999999994+44",
 759                                     "location": "keep:99999999999999999999999999999994+44/file2",
 760                                     "nameext": "",
 761                                     "nameroot": "file2",
 762                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 763                                     "size": 0
 764                                 }
 765                             ],
 766                             "location": "keep:99999999999999999999999999999994+44",
 767                             "path": "/keep/99999999999999999999999999999994+44"
 768                         }
 769                     }},
 770                     'secret_mounts': {},
 771                     'output_storage_classes': ["default"]
 772                 }))
 773
 774     # The test passes no builder.resources
 775     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 776     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 777     def test_secrets(self, keepdocker):
 778         arvados_cwl.add_arv_hints()
 779         runner = mock.MagicMock()
 780         runner.ignore_docker_for_reuse = False
 781         runner.intermediate_output_ttl = 0
 782         runner.secret_store = cwltool.secrets.SecretStore()
 783         runner.api._rootDesc = {"revision": "20210628"}
 784
 785         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 786         runner.api.collections().get().execute.return_value = {
 787             "portable_data_hash": "99999999999999999999999999999993+99"}
 788
 789         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 790
 791         tool = cmap({"arguments": ["md5sum", "example.conf"],
 792                      "class": "CommandLineTool",
 793                      "cwlVersion": "v1.2",
 794                      "hints": [
 795                          {
 796                              "class": "http://commonwl.org/cwltool#Secrets",
 797                              "secrets": [
 798                                  "#secret_job.cwl/pw"
 799                              ]
 800                          }
 801                      ],
 802                      "id": "",
 803                      "inputs": [
 804                          {
 805                              "id": "#secret_job.cwl/pw",
 806                              "type": "string"
 807                          }
 808                      ],
 809                      "outputs": [
 810                      ],
 811                      "requirements": [
 812                          {
 813                              "class": "InitialWorkDirRequirement",
 814                              "listing": [
 815                                  {
 816                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 817                                      "entryname": "example.conf"
 818                                  }
 819                              ]
 820                          }
 821                      ]})
 822
 823         loadingContext, runtimeContext = self.helper(runner)
 824         runtimeContext.name = "test_secrets"
 825
 826         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 827         arvtool.formatgraph = None
 828
 829         job_order = {"pw": "blorp"}
 830         runner.secret_store.store(["pw"], job_order)
 831
 832         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 833             j.run(runtimeContext)
 834             runner.api.container_requests().create.assert_called_with(
 835                 body=JsonDiffMatcher({
 836                     'environment': {
 837                         'HOME': '/var/spool/cwl',
 838                         'TMPDIR': '/tmp'
 839                     },
 840                     'name': 'test_secrets',
 841                     'runtime_constraints': {
 842                         'vcpus': 1,
 843                         'ram': 268435456
 844                     },
 845                     'use_existing': True,
 846                     'priority': 500,
 847                     'mounts': {
 848                         '/tmp': {'kind': 'tmp',
 849                                  "capacity": 1073741824
 850                              },
 851                         '/var/spool/cwl': {'kind': 'tmp',
 852                                            "capacity": 1073741824 }
 853                     },
 854                     'state': 'Committed',
 855                     'output_name': 'Output from step test_secrets',
 856                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 857                     'output_path': '/var/spool/cwl',
 858                     'output_ttl': 0,
 859                     'container_image': '99999999999999999999999999999993+99',
 860                     'command': ['md5sum', 'example.conf'],
 861                     'cwd': '/var/spool/cwl',
 862                     'scheduling_parameters': {},
 863                     'properties': {'cwl_input': job_order},
 864                     "secret_mounts": {
 865                         "/var/spool/cwl/example.conf": {
 866                             "content": "username: user\npassword: blorp\n",
 867                             "kind": "text"
 868                         }
 869                     },
 870                     'output_storage_classes': ["default"]
 871                 }))
 872
 873     # The test passes no builder.resources
 874     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 875     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 876     def test_timelimit(self, keepdocker):
 877         runner = mock.MagicMock()
 878         runner.ignore_docker_for_reuse = False
 879         runner.intermediate_output_ttl = 0
 880         runner.secret_store = cwltool.secrets.SecretStore()
 881         runner.api._rootDesc = {"revision": "20210628"}
 882
 883         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 884         runner.api.collections().get().execute.return_value = {
 885             "portable_data_hash": "99999999999999999999999999999993+99"}
 886
 887         tool = cmap({
 888             "inputs": [],
 889             "outputs": [],
 890             "baseCommand": "ls",
 891             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 892             "id": "",
 893             "cwlVersion": "v1.2",
 894             "class": "CommandLineTool",
 895             "hints": [
 896                 {
 897                     "class": "ToolTimeLimit",
 898                     "timelimit": 42
 899                 }
 900             ]
 901         })
 902
 903         loadingContext, runtimeContext = self.helper(runner)
 904         runtimeContext.name = "test_timelimit"
 905
 906         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 907         arvtool.formatgraph = None
 908
 909         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 910             j.run(runtimeContext)
 911
 912         _, kwargs = runner.api.container_requests().create.call_args
 913         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 914
 915
 916     # The test passes no builder.resources
 917     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 918     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 919     def test_setting_storage_class(self, keepdocker):
 920         arv_docker_clear_cache()
 921
 922         runner = mock.MagicMock()
 923         runner.ignore_docker_for_reuse = False
 924         runner.intermediate_output_ttl = 0
 925         runner.secret_store = cwltool.secrets.SecretStore()
 926         runner.api._rootDesc = {"revision": "20210628"}
 927
 928         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 929         runner.api.collections().get().execute.return_value = {
 930             "portable_data_hash": "99999999999999999999999999999993+99"}
 931
 932         tool = cmap({
 933             "inputs": [],
 934             "outputs": [],
 935             "baseCommand": "ls",
 936             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 937             "id": "",
 938             "cwlVersion": "v1.2",
 939             "class": "CommandLineTool",
 940             "hints": [
 941                 {
 942                     "class": "http://arvados.org/cwl#OutputStorageClass",
 943                     "finalStorageClass": ["baz_sc", "qux_sc"],
 944                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 945                 }
 946             ]
 947         })
 948
 949         loadingContext, runtimeContext = self.helper(runner, True)
 950
 951         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 952         arvtool.formatgraph = None
 953
 954         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 955             j.run(runtimeContext)
 956             runner.api.container_requests().create.assert_called_with(
 957                 body=JsonDiffMatcher({
 958                     'environment': {
 959                         'HOME': '/var/spool/cwl',
 960                         'TMPDIR': '/tmp'
 961                     },
 962                     'name': 'test_run_True',
 963                     'runtime_constraints': {
 964                         'vcpus': 1,
 965                         'ram': 268435456
 966                     },
 967                     'use_existing': True,
 968                     'priority': 500,
 969                     'mounts': {
 970                         '/tmp': {'kind': 'tmp',
 971                                  "capacity": 1073741824
 972                              },
 973                         '/var/spool/cwl': {'kind': 'tmp',
 974                                            "capacity": 1073741824 }
 975                     },
 976                     'state': 'Committed',
 977                     'output_name': 'Output from step test_run_True',
 978                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 979                     'output_path': '/var/spool/cwl',
 980                     'output_ttl': 0,
 981                     'container_image': '99999999999999999999999999999993+99',
 982                     'command': ['ls', '/var/spool/cwl'],
 983                     'cwd': '/var/spool/cwl',
 984                     'scheduling_parameters': {},
 985                     'properties': {'cwl_input': {}},
 986                     'secret_mounts': {},
 987                     'output_storage_classes': ["foo_sc", "bar_sc"]
 988                 }))
 989
 990
 991     # The test passes no builder.resources
 992     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 993     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 994     def test_setting_process_properties(self, keepdocker):
 995         arv_docker_clear_cache()
 996
 997         runner = mock.MagicMock()
 998         runner.ignore_docker_for_reuse = False
 999         runner.intermediate_output_ttl = 0
1000         runner.secret_store = cwltool.secrets.SecretStore()
1001         runner.api._rootDesc = {"revision": "20210628"}
1002
1003         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1004         runner.api.collections().get().execute.return_value = {
1005             "portable_data_hash": "99999999999999999999999999999993+99"}
1006
1007         tool = cmap({
1008             "inputs": [
1009                 {"id": "x", "type": "string"}],
1010             "outputs": [],
1011             "baseCommand": "ls",
1012             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1013             "id": "",
1014             "class": "CommandLineTool",
1015             "cwlVersion": "v1.2",
1016             "hints": [
1017             {
1018                 "class": "http://arvados.org/cwl#ProcessProperties",
1019                 "processProperties": [
1020                     {"propertyName": "foo",
1021                      "propertyValue": "bar"},
1022                     {"propertyName": "baz",
1023                      "propertyValue": "$(inputs.x)"},
1024                     {"propertyName": "quux",
1025                      "propertyValue": {
1026                          "q1": 1,
1027                          "q2": 2
1028                      }
1029                     }
1030                 ],
1031             }
1032         ]
1033         })
1034
1035         loadingContext, runtimeContext = self.helper(runner, True)
1036
1037         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1038         arvtool.formatgraph = None
1039
1040         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1041             j.run(runtimeContext)
1042             runner.api.container_requests().create.assert_called_with(
1043                 body=JsonDiffMatcher({
1044                     'environment': {
1045                         'HOME': '/var/spool/cwl',
1046                         'TMPDIR': '/tmp'
1047                     },
1048                     'name': 'test_run_True',
1049                     'runtime_constraints': {
1050                         'vcpus': 1,
1051                         'ram': 268435456
1052                     },
1053                     'use_existing': True,
1054                     'priority': 500,
1055                     'mounts': {
1056                         '/tmp': {'kind': 'tmp',
1057                                  "capacity": 1073741824
1058                              },
1059                         '/var/spool/cwl': {'kind': 'tmp',
1060                                            "capacity": 1073741824 }
1061                     },
1062                     'state': 'Committed',
1063                     'output_name': 'Output from step test_run_True',
1064                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1065                     'output_path': '/var/spool/cwl',
1066                     'output_ttl': 0,
1067                     'container_image': '99999999999999999999999999999993+99',
1068                     'command': ['ls', '/var/spool/cwl'],
1069                     'cwd': '/var/spool/cwl',
1070                     'scheduling_parameters': {},
1071                     'properties': {
1072                         "baz": "blorp",
1073                         "cwl_input": {"x": "blorp"},
1074                         "foo": "bar",
1075                         "quux": {
1076                             "q1": 1,
1077                             "q2": 2
1078                         }
1079                     },
1080                     'secret_mounts': {},
1081                     'output_storage_classes': ["default"]
1082                 }))
1083
1084
1085     # The test passes no builder.resources
1086     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1087     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1088     def test_cuda_requirement(self, keepdocker):
1089         arvados_cwl.add_arv_hints()
1090         arv_docker_clear_cache()
1091
1092         runner = mock.MagicMock()
1093         runner.ignore_docker_for_reuse = False
1094         runner.intermediate_output_ttl = 0
1095         runner.secret_store = cwltool.secrets.SecretStore()
1096         runner.api._rootDesc = {"revision": "20210628"}
1097
1098         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1099         runner.api.collections().get().execute.return_value = {
1100             "portable_data_hash": "99999999999999999999999999999993+99"}
1101
1102         test_cwl_req = [{
1103                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1104                 "cudaVersionMin": "11.0",
1105                 "cudaComputeCapability": "9.0",
1106             }, {
1107                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1108                 "cudaVersionMin": "11.0",
1109                 "cudaComputeCapability": "9.0",
1110                 "cudaDeviceCountMin": 2
1111             }, {
1112                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1113                 "cudaVersionMin": "11.0",
1114                 "cudaComputeCapability": ["4.0", "5.0"],
1115                 "cudaDeviceCountMin": 2
1116             }]
1117
1118         test_arv_req = [{
1119             'device_count': 1,
1120             'driver_version': "11.0",
1121             'hardware_capability': "9.0"
1122         }, {
1123             'device_count': 2,
1124             'driver_version': "11.0",
1125             'hardware_capability': "9.0"
1126         }, {
1127             'device_count': 2,
1128             'driver_version': "11.0",
1129             'hardware_capability': "4.0"
1130         }]
1131
1132         for test_case in range(0, len(test_cwl_req)):
1133
1134             tool = cmap({
1135                 "inputs": [],
1136                 "outputs": [],
1137                 "baseCommand": "nvidia-smi",
1138                 "arguments": [],
1139                 "id": "",
1140                 "cwlVersion": "v1.2",
1141                 "class": "CommandLineTool",
1142                 "requirements": [test_cwl_req[test_case]]
1143             })
1144
1145             loadingContext, runtimeContext = self.helper(runner, True)
1146
1147             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1148             arvtool.formatgraph = None
1149
1150             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1151                 j.run(runtimeContext)
1152                 runner.api.container_requests().create.assert_called_with(
1153                     body=JsonDiffMatcher({
1154                         'environment': {
1155                             'HOME': '/var/spool/cwl',
1156                             'TMPDIR': '/tmp'
1157                         },
1158                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1159                         'runtime_constraints': {
1160                             'vcpus': 1,
1161                             'ram': 268435456,
1162                             'cuda': test_arv_req[test_case]
1163                         },
1164                         'use_existing': True,
1165                         'priority': 500,
1166                         'mounts': {
1167                             '/tmp': {'kind': 'tmp',
1168                                      "capacity": 1073741824
1169                                  },
1170                             '/var/spool/cwl': {'kind': 'tmp',
1171                                                "capacity": 1073741824 }
1172                         },
1173                         'state': 'Committed',
1174                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1175                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1176                         'output_path': '/var/spool/cwl',
1177                         'output_ttl': 0,
1178                         'container_image': '99999999999999999999999999999993+99',
1179                         'command': ['nvidia-smi'],
1180                         'cwd': '/var/spool/cwl',
1181                         'scheduling_parameters': {},
1182                         'properties': {'cwl_input': {}},
1183                         'secret_mounts': {},
1184                         'output_storage_classes': ["default"]
1185                     }))
1186
1187
1188     # The test passes no builder.resources
1189     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1190     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1191     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1192     def test_match_local_docker(self, keepdocker, determine_image_id):
1193         arvados_cwl.add_arv_hints()
1194         arv_docker_clear_cache()
1195
1196         runner = mock.MagicMock()
1197         runner.ignore_docker_for_reuse = False
1198         runner.intermediate_output_ttl = 0
1199         runner.secret_store = cwltool.secrets.SecretStore()
1200         runner.api._rootDesc = {"revision": "20210628"}
1201
1202         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1203                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1204         determine_image_id.side_effect = lambda x: "123"
1205         def execute(uuid):
1206             ex = mock.MagicMock()
1207             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1208                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1209             ex.execute.return_value = lookup[uuid]
1210             return ex
1211         runner.api.collections().get.side_effect = execute
1212
1213         tool = cmap({
1214             "inputs": [],
1215             "outputs": [],
1216             "baseCommand": "echo",
1217             "arguments": [],
1218             "id": "",
1219             "cwlVersion": "v1.0",
1220             "class": "org.w3id.cwl.cwl.CommandLineTool"
1221         })
1222
1223         loadingContext, runtimeContext = self.helper(runner, True)
1224
1225         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1226         arvtool.formatgraph = None
1227
1228         container_request = {
1229             'environment': {
1230                 'HOME': '/var/spool/cwl',
1231                 'TMPDIR': '/tmp'
1232             },
1233             'name': 'test_run_True',
1234             'runtime_constraints': {
1235                 'vcpus': 1,
1236                 'ram': 1073741824,
1237             },
1238             'use_existing': True,
1239             'priority': 500,
1240             'mounts': {
1241                 '/tmp': {'kind': 'tmp',
1242                          "capacity": 1073741824
1243                          },
1244                 '/var/spool/cwl': {'kind': 'tmp',
1245                                    "capacity": 1073741824 }
1246             },
1247             'state': 'Committed',
1248             'output_name': 'Output from step test_run_True',
1249             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1250             'output_path': '/var/spool/cwl',
1251             'output_ttl': 0,
1252             'container_image': '99999999999999999999999999999994+99',
1253             'command': ['echo'],
1254             'cwd': '/var/spool/cwl',
1255             'scheduling_parameters': {},
1256             'properties': {'cwl_input': {}},
1257             'secret_mounts': {},
1258             'output_storage_classes': ["default"]
1259         }
1260
1261         runtimeContext.match_local_docker = False
1262         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1263             j.run(runtimeContext)
1264             runner.api.container_requests().create.assert_called_with(
1265                 body=JsonDiffMatcher(container_request))
1266
1267         arv_docker_clear_cache()
1268         runtimeContext.match_local_docker = True
1269         container_request['container_image'] = '99999999999999999999999999999993+99'
1270         container_request['name'] = 'test_run_True_2'
1271         container_request['output_name'] = 'Output from step test_run_True_2'
1272         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1273             j.run(runtimeContext)
1274             runner.api.container_requests().create.assert_called_with(
1275                 body=JsonDiffMatcher(container_request))
1276
1277
1278     # The test passes no builder.resources
1279     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1280     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1281     def test_run_preemptible_hint(self, keepdocker):
1282         arvados_cwl.add_arv_hints()
1283         for enable_preemptible in (None, True, False):
1284             for preemptible_hint in (None, True, False):
1285                 arv_docker_clear_cache()
1286
1287                 runner = mock.MagicMock()
1288                 runner.ignore_docker_for_reuse = False
1289                 runner.intermediate_output_ttl = 0
1290                 runner.secret_store = cwltool.secrets.SecretStore()
1291                 runner.api._rootDesc = {"revision": "20210628"}
1292
1293                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1294                 runner.api.collections().get().execute.return_value = {
1295                     "portable_data_hash": "99999999999999999999999999999993+99"}
1296
1297                 if preemptible_hint is not None:
1298                     hints = [{
1299                         "class": "http://arvados.org/cwl#UsePreemptible",
1300                         "usePreemptible": preemptible_hint
1301                     }]
1302                 else:
1303                     hints = []
1304
1305                 tool = cmap({
1306                     "inputs": [],
1307                     "outputs": [],
1308                     "baseCommand": "ls",
1309                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1310                     "id": "",
1311                     "class": "CommandLineTool",
1312                     "cwlVersion": "v1.2",
1313                     "hints": hints
1314                 })
1315
1316                 loadingContext, runtimeContext = self.helper(runner)
1317
1318                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1319                 runtimeContext.enable_preemptible = enable_preemptible
1320
1321                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1322                 arvtool.formatgraph = None
1323
1324                 # Test the interactions between --enable/disable-preemptible
1325                 # and UsePreemptible hint
1326
1327                 if enable_preemptible is None:
1328                     if preemptible_hint is None:
1329                         sched = {}
1330                     else:
1331                         sched = {'preemptible': preemptible_hint}
1332                 else:
1333                     if preemptible_hint is None:
1334                         sched = {'preemptible': enable_preemptible}
1335                     else:
1336                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1337
1338                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1339                     j.run(runtimeContext)
1340                     runner.api.container_requests().create.assert_called_with(
1341                         body=JsonDiffMatcher({
1342                             'environment': {
1343                                 'HOME': '/var/spool/cwl',
1344                                 'TMPDIR': '/tmp'
1345                             },
1346                             'name': runtimeContext.name,
1347                             'runtime_constraints': {
1348                                 'vcpus': 1,
1349                                 'ram': 268435456
1350                             },
1351                             'use_existing': True,
1352                             'priority': 500,
1353                             'mounts': {
1354                                 '/tmp': {'kind': 'tmp',
1355                                          "capacity": 1073741824
1356                                      },
1357                                 '/var/spool/cwl': {'kind': 'tmp',
1358                                                    "capacity": 1073741824 }
1359                             },
1360                             'state': 'Committed',
1361                             'output_name': 'Output from step '+runtimeContext.name,
1362                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1363                             'output_path': '/var/spool/cwl',
1364                             'output_ttl': 0,
1365                             'container_image': '99999999999999999999999999999993+99',
1366                             'command': ['ls', '/var/spool/cwl'],
1367                             'cwd': '/var/spool/cwl',
1368                             'scheduling_parameters': sched,
1369                             'properties': {'cwl_input': {}},
1370                             'secret_mounts': {},
1371                             'output_storage_classes': ["default"]
1372                         }))
1373
1374
1375     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1376     def test_output_properties(self, keepdocker):
1377         arvados_cwl.add_arv_hints()
1378         for rev in ["20210628", "20220510"]:
1379             runner = mock.MagicMock()
1380             runner.ignore_docker_for_reuse = False
1381             runner.intermediate_output_ttl = 0
1382             runner.secret_store = cwltool.secrets.SecretStore()
1383             runner.api._rootDesc = {"revision": rev}
1384
1385             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1386             runner.api.collections().get().execute.return_value = {
1387                 "portable_data_hash": "99999999999999999999999999999993+99"}
1388
1389             tool = cmap({
1390                 "inputs": [{
1391                     "id": "inp",
1392                     "type": "string"
1393                 }],
1394                 "outputs": [],
1395                 "baseCommand": "ls",
1396                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1397                 "id": "",
1398                 "cwlVersion": "v1.2",
1399                 "class": "CommandLineTool",
1400                 "hints": [
1401                     {
1402                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1403                         "outputProperties": {
1404                             "foo": "bar",
1405                             "baz": "$(inputs.inp)"
1406                         }
1407                     }
1408                 ]
1409             })
1410
1411             loadingContext, runtimeContext = self.helper(runner)
1412             runtimeContext.name = "test_timelimit"
1413
1414             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1415             arvtool.formatgraph = None
1416
1417             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1418                 j.run(runtimeContext)
1419
1420             _, kwargs = runner.api.container_requests().create.call_args
1421             if rev == "20220510":
1422                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1423             else:
1424                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1425
1426
1427 class TestWorkflow(unittest.TestCase):
1428     def setUp(self):
1429         cwltool.process._names = set()
1430         arv_docker_clear_cache()
1431
1432     def helper(self, runner, enable_reuse=True):
1433         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1434
1435         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1436                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1437
1438         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1439         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1440         document_loader.fetch_text = document_loader.fetcher.fetch_text
1441         document_loader.check_exists = document_loader.fetcher.check_exists
1442
1443         loadingContext = arvados_cwl.context.ArvLoadingContext(
1444             {"avsc_names": avsc_names,
1445              "basedir": "",
1446              "make_fs_access": make_fs_access,
1447              "loader": document_loader,
1448              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1449              "construct_tool_object": runner.arv_make_tool})
1450         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1451             {"work_api": "containers",
1452              "basedir": "",
1453              "name": "test_run_wf_"+str(enable_reuse),
1454              "make_fs_access": make_fs_access,
1455              "tmpdir": "/tmp",
1456              "enable_reuse": enable_reuse,
1457              "priority": 500})
1458
1459         return loadingContext, runtimeContext
1460
1461     # The test passes no builder.resources
1462     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1463     @mock.patch("arvados.collection.CollectionReader")
1464     @mock.patch("arvados.collection.Collection")
1465     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1466     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1467         arvados_cwl.add_arv_hints()
1468
1469         api = mock.MagicMock()
1470         api._rootDesc = get_rootDesc()
1471
1472         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1473         self.assertEqual(runner.work_api, 'containers')
1474
1475         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1476         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1477         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1478                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1479
1480         runner.api.containers().current().execute.return_value = {}
1481
1482         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1483         runner.ignore_docker_for_reuse = False
1484         runner.num_retries = 0
1485         runner.secret_store = cwltool.secrets.SecretStore()
1486
1487         loadingContext, runtimeContext = self.helper(runner)
1488         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1489
1490         mockcollectionreader().exists.return_value = True
1491
1492         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1493         metadata["cwlVersion"] = tool["cwlVersion"]
1494
1495         mockc = mock.MagicMock()
1496         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1497         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1498
1499         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1500         arvtool.formatgraph = None
1501         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1502
1503         next(it).run(runtimeContext)
1504         next(it).run(runtimeContext)
1505
1506         with open("tests/wf/scatter2_subwf.cwl") as f:
1507             subwf = StripYAMLComments(f.read()).rstrip()
1508
1509         runner.api.container_requests().create.assert_called_with(
1510             body=JsonDiffMatcher({
1511                 "command": [
1512                     "cwltool",
1513                     "--no-container",
1514                     "--move-outputs",
1515                     "--preserve-entire-environment",
1516                     "workflow.cwl",
1517                     "cwl.input.yml"
1518                 ],
1519                 "container_image": "99999999999999999999999999999993+99",
1520                 "cwd": "/var/spool/cwl",
1521                 "environment": {
1522                     "HOME": "/var/spool/cwl",
1523                     "TMPDIR": "/tmp"
1524                 },
1525                 "mounts": {
1526                     "/keep/99999999999999999999999999999999+118": {
1527                         "kind": "collection",
1528                         "portable_data_hash": "99999999999999999999999999999999+118"
1529                     },
1530                     "/tmp": {
1531                         "capacity": 1073741824,
1532                         "kind": "tmp"
1533                     },
1534                     "/var/spool/cwl": {
1535                         "capacity": 1073741824,
1536                         "kind": "tmp"
1537                     },
1538                     "/var/spool/cwl/cwl.input.yml": {
1539                         "kind": "collection",
1540                         "path": "cwl.input.yml",
1541                         "portable_data_hash": "99999999999999999999999999999996+99"
1542                     },
1543                     "/var/spool/cwl/workflow.cwl": {
1544                         "kind": "collection",
1545                         "path": "workflow.cwl",
1546                         "portable_data_hash": "99999999999999999999999999999996+99"
1547                     },
1548                     "stdout": {
1549                         "kind": "file",
1550                         "path": "/var/spool/cwl/cwl.output.json"
1551                     }
1552                 },
1553                 "name": "scatterstep",
1554                 "output_name": "Output from step scatterstep",
1555                 "output_path": "/var/spool/cwl",
1556                 "output_ttl": 0,
1557                 "priority": 500,
1558                 "properties": {'cwl_input': {
1559                         "fileblub": {
1560                             "basename": "token.txt",
1561                             "class": "File",
1562                             "dirname": "/keep/99999999999999999999999999999999+118",
1563                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1564                             "nameext": ".txt",
1565                             "nameroot": "token",
1566                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1567                             "size": 0
1568                         },
1569                         "sleeptime": 5
1570                 }},
1571                 "runtime_constraints": {
1572                     "ram": 1073741824,
1573                     "vcpus": 1
1574                 },
1575                 "scheduling_parameters": {},
1576                 "secret_mounts": {},
1577                 "state": "Committed",
1578                 "use_existing": True,
1579                 'output_storage_classes': ["default"]
1580             }))
1581         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1582         mockc.open().__enter__().write.assert_has_calls([mock.call(
1583 '''{
1584   "fileblub": {
1585     "basename": "token.txt",
1586     "class": "File",
1587     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1588     "size": 0
1589   },
1590   "sleeptime": 5
1591 }''')])
1592
1593     # The test passes no builder.resources
1594     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1595     @mock.patch("arvados.collection.CollectionReader")
1596     @mock.patch("arvados.collection.Collection")
1597     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1598     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1599         arvados_cwl.add_arv_hints()
1600
1601         api = mock.MagicMock()
1602         api._rootDesc = get_rootDesc()
1603
1604         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1605         self.assertEqual(runner.work_api, 'containers')
1606
1607         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1608         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1609                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1610         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1611                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1612
1613         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1614         runner.ignore_docker_for_reuse = False
1615         runner.num_retries = 0
1616         runner.secret_store = cwltool.secrets.SecretStore()
1617
1618         loadingContext, runtimeContext = self.helper(runner)
1619         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1620         loadingContext.do_update = True
1621         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1622
1623         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1624
1625         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1626         arvtool.formatgraph = None
1627         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1628
1629         next(it).run(runtimeContext)
1630         next(it).run(runtimeContext)
1631
1632         with open("tests/wf/echo-subwf.cwl") as f:
1633             subwf = StripYAMLComments(f.read())
1634
1635         runner.api.container_requests().create.assert_called_with(
1636             body=JsonDiffMatcher({
1637                 'output_ttl': 0,
1638                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1639                 'scheduling_parameters': {},
1640                 'name': u'echo-subwf',
1641                 'secret_mounts': {},
1642                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1643                 'properties': {'cwl_input': {}},
1644                 'priority': 500,
1645                 'mounts': {
1646                     '/var/spool/cwl/cwl.input.yml': {
1647                         'portable_data_hash': '99999999999999999999999999999996+99',
1648                         'kind': 'collection',
1649                         'path': 'cwl.input.yml'
1650                     },
1651                     '/var/spool/cwl/workflow.cwl': {
1652                         'portable_data_hash': '99999999999999999999999999999996+99',
1653                         'kind': 'collection',
1654                         'path': 'workflow.cwl'
1655                     },
1656                     'stdout': {
1657                         'path': '/var/spool/cwl/cwl.output.json',
1658                         'kind': 'file'
1659                     },
1660                     '/tmp': {
1661                         'kind': 'tmp',
1662                         'capacity': 1073741824
1663                     }, '/var/spool/cwl': {
1664                         'kind': 'tmp',
1665                         'capacity': 3221225472
1666                     }
1667                 },
1668                 'state': 'Committed',
1669                 'output_path': '/var/spool/cwl',
1670                 'container_image': '99999999999999999999999999999993+99',
1671                 'command': [
1672                     u'cwltool',
1673                     u'--no-container',
1674                     u'--move-outputs',
1675                     u'--preserve-entire-environment',
1676                     u'workflow.cwl',
1677                     u'cwl.input.yml'
1678                 ],
1679                 'use_existing': True,
1680                 'output_name': u'Output from step echo-subwf',
1681                 'cwd': '/var/spool/cwl',
1682                 'output_storage_classes': ["default"]
1683             }))
1684
1685     def test_default_work_api(self):
1686         arvados_cwl.add_arv_hints()
1687
1688         api = mock.MagicMock()
1689         api._rootDesc = copy.deepcopy(get_rootDesc())
1690         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1691         self.assertEqual(runner.work_api, 'containers')