sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         tool = cmap({
 210             "inputs": [],
 211             "outputs": [],
 212             "hints": [{
 213                 "class": "ResourceRequirement",
 214                 "coresMin": 3,
 215                 "ramMin": 3000,
 216                 "tmpdirMin": 4000,
 217                 "outdirMin": 5000
 218             }, {
 219                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 220                 "keep_cache": 512
 221             }, {
 222                 "class": "http://arvados.org/cwl#APIRequirement",
 223             }, {
 224                 "class": "http://arvados.org/cwl#PartitionRequirement",
 225                 "partition": "blurb"
 226             }, {
 227                 "class": "http://arvados.org/cwl#IntermediateOutput",
 228                 "outputTTL": 7200
 229             }, {
 230                 "class": "WorkReuse",
 231                 "enableReuse": False
 232             }],
 233             "baseCommand": "ls",
 234             "id": "",
 235             "class": "CommandLineTool",
 236             "cwlVersion": "v1.2"
 237         })
 238
 239         loadingContext, runtimeContext = self.helper(runner)
 240         runtimeContext.name = "test_resource_requirements"
 241
 242         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 243         arvtool.formatgraph = None
 244         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 245             j.run(runtimeContext)
 246
 247         call_args, call_kwargs = runner.api.container_requests().create.call_args
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_resource_requirements',
 255             'runtime_constraints': {
 256                 'vcpus': 3,
 257                 'ram': 3145728000,
 258                 'keep_cache_ram': 536870912,
 259                 'API': True
 260             },
 261             'use_existing': False,
 262             'priority': 500,
 263             'mounts': {
 264                 '/tmp': {'kind': 'tmp',
 265                          "capacity": 4194304000 },
 266                 '/var/spool/cwl': {'kind': 'tmp',
 267                                    "capacity": 5242880000 }
 268             },
 269             'state': 'Committed',
 270             'output_name': 'Output from step test_resource_requirements',
 271             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272             'output_path': '/var/spool/cwl',
 273             'output_ttl': 7200,
 274             'container_image': '99999999999999999999999999999993+99',
 275             'command': ['ls'],
 276             'cwd': '/var/spool/cwl',
 277             'scheduling_parameters': {
 278                 'partitions': ['blurb']
 279             },
 280             'properties': {},
 281             'secret_mounts': {},
 282             'output_storage_classes': ["default"]
 283         }
 284
 285         call_body = call_kwargs.get('body', None)
 286         self.assertNotEqual(None, call_body)
 287         for key in call_body:
 288             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 289
 290
 291     # The test passes some fields in builder.resources
 292     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 293     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 294     @mock.patch("arvados.collection.Collection")
 295     def test_initial_work_dir(self, collection_mock, keepdocker):
 296         runner = mock.MagicMock()
 297         runner.ignore_docker_for_reuse = False
 298         runner.intermediate_output_ttl = 0
 299         runner.secret_store = cwltool.secrets.SecretStore()
 300         runner.api._rootDesc = {"revision": "20210628"}
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         sourcemock = mock.MagicMock()
 307         def get_collection_mock(p):
 308             if "/" in p:
 309                 return (sourcemock, p.split("/", 1)[1])
 310             else:
 311                 return (sourcemock, "")
 312         runner.fs_access.get_collection.side_effect = get_collection_mock
 313
 314         vwdmock = mock.MagicMock()
 315         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 316
 317         tool = cmap({
 318             "inputs": [],
 319             "outputs": [],
 320             "hints": [{
 321                 "class": "InitialWorkDirRequirement",
 322                 "listing": [{
 323                     "class": "File",
 324                     "basename": "foo",
 325                     "location": "keep:99999999999999999999999999999995+99/bar"
 326                 },
 327                 {
 328                     "class": "Directory",
 329                     "basename": "foo2",
 330                     "location": "keep:99999999999999999999999999999995+99"
 331                 },
 332                 {
 333                     "class": "File",
 334                     "basename": "filename",
 335                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 336                 },
 337                 {
 338                     "class": "Directory",
 339                     "basename": "subdir",
 340                     "location": "keep:99999999999999999999999999999995+99/subdir"
 341                 }                        ]
 342             }],
 343             "baseCommand": "ls",
 344             "class": "CommandLineTool",
 345             "cwlVersion": "v1.2",
 346             "id": ""
 347         })
 348
 349         loadingContext, runtimeContext = self.helper(runner)
 350         runtimeContext.name = "test_initial_work_dir"
 351
 352         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 353
 354         arvtool.formatgraph = None
 355         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 356             j.run(runtimeContext)
 357
 358         call_args, call_kwargs = runner.api.container_requests().create.call_args
 359
 360         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 361         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 364
 365         call_body_expected = {
 366             'environment': {
 367                 'HOME': '/var/spool/cwl',
 368                 'TMPDIR': '/tmp'
 369             },
 370             'name': 'test_initial_work_dir',
 371             'runtime_constraints': {
 372                 'vcpus': 1,
 373                 'ram': 268435456
 374             },
 375             'use_existing': True,
 376             'priority': 500,
 377             'mounts': {
 378                 '/tmp': {'kind': 'tmp',
 379                          "capacity": 1073741824 },
 380                 '/var/spool/cwl': {'kind': 'tmp',
 381                                    "capacity": 1073741824 },
 382                 '/var/spool/cwl/foo': {
 383                     'kind': 'collection',
 384                     'path': 'foo',
 385                     'portable_data_hash': '99999999999999999999999999999996+99'
 386                 },
 387                 '/var/spool/cwl/foo2': {
 388                     'kind': 'collection',
 389                     'path': 'foo2',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/filename': {
 393                     'kind': 'collection',
 394                     'path': 'filename',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/subdir': {
 398                     'kind': 'collection',
 399                     'path': 'subdir',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 }
 402             },
 403             'state': 'Committed',
 404             'output_name': 'Output from step test_initial_work_dir',
 405             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 406             'output_path': '/var/spool/cwl',
 407             'output_ttl': 0,
 408             'container_image': '99999999999999999999999999999993+99',
 409             'command': ['ls'],
 410             'cwd': '/var/spool/cwl',
 411             'scheduling_parameters': {
 412             },
 413             'properties': {},
 414             'secret_mounts': {},
 415             'output_storage_classes': ["default"]
 416         }
 417
 418         call_body = call_kwargs.get('body', None)
 419         self.assertNotEqual(None, call_body)
 420         for key in call_body:
 421             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 422
 423
 424     # Test redirecting stdin/stdout/stderr
 425     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 426     def test_redirects(self, keepdocker):
 427         runner = mock.MagicMock()
 428         runner.ignore_docker_for_reuse = False
 429         runner.intermediate_output_ttl = 0
 430         runner.secret_store = cwltool.secrets.SecretStore()
 431         runner.api._rootDesc = {"revision": "20210628"}
 432
 433         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 434         runner.api.collections().get().execute.return_value = {
 435             "portable_data_hash": "99999999999999999999999999999993+99"}
 436
 437         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 438
 439         tool = cmap({
 440             "inputs": [],
 441             "outputs": [],
 442             "baseCommand": "ls",
 443             "stdout": "stdout.txt",
 444             "stderr": "stderr.txt",
 445             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 446             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 447             "id": "",
 448             "class": "CommandLineTool",
 449             "cwlVersion": "v1.2"
 450         })
 451
 452         loadingContext, runtimeContext = self.helper(runner)
 453         runtimeContext.name = "test_run_redirect"
 454
 455         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 456         arvtool.formatgraph = None
 457         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 458             j.run(runtimeContext)
 459             runner.api.container_requests().create.assert_called_with(
 460                 body=JsonDiffMatcher({
 461                     'environment': {
 462                         'HOME': '/var/spool/cwl',
 463                         'TMPDIR': '/tmp'
 464                     },
 465                     'name': 'test_run_redirect',
 466                     'runtime_constraints': {
 467                         'vcpus': 1,
 468                         'ram': 268435456
 469                     },
 470                     'use_existing': True,
 471                     'priority': 500,
 472                     'mounts': {
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 },
 477                         "stderr": {
 478                             "kind": "file",
 479                             "path": "/var/spool/cwl/stderr.txt"
 480                         },
 481                         "stdin": {
 482                             "kind": "collection",
 483                             "path": "file.txt",
 484                             "portable_data_hash": "99999999999999999999999999999996+99"
 485                         },
 486                         "stdout": {
 487                             "kind": "file",
 488                             "path": "/var/spool/cwl/stdout.txt"
 489                         },
 490                     },
 491                     'state': 'Committed',
 492                     "output_name": "Output from step test_run_redirect",
 493                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 494                     'output_path': '/var/spool/cwl',
 495                     'output_ttl': 0,
 496                     'container_image': '99999999999999999999999999999993+99',
 497                     'command': ['ls', '/var/spool/cwl'],
 498                     'cwd': '/var/spool/cwl',
 499                     'scheduling_parameters': {},
 500                     'properties': {},
 501                     'secret_mounts': {},
 502                     'output_storage_classes': ["default"]
 503                 }))
 504
 505     @mock.patch("arvados.collection.Collection")
 506     def test_done(self, col):
 507         api = mock.MagicMock()
 508
 509         runner = mock.MagicMock()
 510         runner.api = api
 511         runner.num_retries = 0
 512         runner.ignore_docker_for_reuse = False
 513         runner.intermediate_output_ttl = 0
 514         runner.secret_store = cwltool.secrets.SecretStore()
 515
 516         runner.api.containers().get().execute.return_value = {"state":"Complete",
 517                                                               "output": "abc+123",
 518                                                               "exit_code": 0}
 519
 520         col().open.return_value = []
 521
 522         loadingContext, runtimeContext = self.helper(runner)
 523
 524         arvjob = arvados_cwl.ArvadosContainer(runner,
 525                                               runtimeContext,
 526                                               mock.MagicMock(),
 527                                               {},
 528                                               None,
 529                                               [],
 530                                               [],
 531                                               "testjob")
 532         arvjob.output_callback = mock.MagicMock()
 533         arvjob.collect_outputs = mock.MagicMock()
 534         arvjob.successCodes = [0]
 535         arvjob.outdir = "/var/spool/cwl"
 536         arvjob.output_ttl = 3600
 537
 538         arvjob.collect_outputs.return_value = {"out": "stuff"}
 539
 540         arvjob.done({
 541             "state": "Final",
 542             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 543             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 544             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 545             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 546             "modified_at": "2017-05-26T12:01:22Z"
 547         })
 548
 549         self.assertFalse(api.collections().create.called)
 550         self.assertFalse(runner.runtime_status_error.called)
 551
 552         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 553         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 554         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 555
 556     # Test to make sure we dont call runtime_status_update if we already did
 557     # some where higher up in the call stack
 558     @mock.patch("arvados_cwl.util.get_current_container")
 559     def test_recursive_runtime_status_update(self, gcc_mock):
 560         self.setup_and_test_container_executor_and_logging(gcc_mock)
 561         root_logger = logging.getLogger('')
 562
 563         # get_current_container is invoked when we call runtime_status_update
 564         # so try and log again!
 565         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 566         try:
 567             root_logger.error("First Error")
 568         except RuntimeError:
 569             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 570
 571
 572     # Test to make sure that an exception raised from
 573     # get_current_container doesn't cause the logger to raise an
 574     # exception
 575     @mock.patch("arvados_cwl.util.get_current_container")
 576     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 577         self.setup_and_test_container_executor_and_logging(gcc_mock)
 578         root_logger = logging.getLogger('')
 579
 580         # get_current_container is invoked when we call
 581         # runtime_status_update, it is going to also raise an
 582         # exception.
 583         gcc_mock.side_effect = Exception("Second Error")
 584         try:
 585             root_logger.error("First Error")
 586         except Exception:
 587             self.fail("Exception in logger should not propagate")
 588         self.assertTrue(gcc_mock.called)
 589
 590     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 591     @mock.patch("arvados_cwl.util.get_current_container")
 592     @mock.patch("arvados.collection.CollectionReader")
 593     @mock.patch("arvados.collection.Collection")
 594     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 595         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 596
 597         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 598         self.assertTrue(gcc_mock.called)
 599
 600         runner.num_retries = 0
 601         runner.ignore_docker_for_reuse = False
 602         runner.intermediate_output_ttl = 0
 603         runner.secret_store = cwltool.secrets.SecretStore()
 604         runner.label = mock.MagicMock()
 605         runner.label.return_value = '[container testjob]'
 606
 607         runner.api.containers().get().execute.return_value = {
 608             "state":"Complete",
 609             "output": "abc+123",
 610             "exit_code": 1,
 611             "log": "def+234"
 612         }
 613
 614         col().open.return_value = []
 615
 616         loadingContext, runtimeContext = self.helper(runner)
 617
 618         arvjob = arvados_cwl.ArvadosContainer(runner,
 619                                               runtimeContext,
 620                                               mock.MagicMock(),
 621                                               {},
 622                                               None,
 623                                               [],
 624                                               [],
 625                                               "testjob")
 626         arvjob.output_callback = mock.MagicMock()
 627         arvjob.collect_outputs = mock.MagicMock()
 628         arvjob.successCodes = [0]
 629         arvjob.outdir = "/var/spool/cwl"
 630         arvjob.output_ttl = 3600
 631         arvjob.collect_outputs.return_value = {"out": "stuff"}
 632
 633         arvjob.done({
 634             "state": "Final",
 635             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 636             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 637             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 638             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 639             "modified_at": "2017-05-26T12:01:22Z"
 640         })
 641
 642         rts_mock.assert_called_with(
 643             'error',
 644             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 645             '  ** log is empty **'
 646         )
 647         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 648
 649     # The test passes no builder.resources
 650     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 651     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 652     def test_mounts(self, keepdocker):
 653         runner = mock.MagicMock()
 654         runner.ignore_docker_for_reuse = False
 655         runner.intermediate_output_ttl = 0
 656         runner.secret_store = cwltool.secrets.SecretStore()
 657         runner.api._rootDesc = {"revision": "20210628"}
 658
 659         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 660         runner.api.collections().get().execute.return_value = {
 661             "portable_data_hash": "99999999999999999999999999999994+99",
 662             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 663
 664         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 665
 666         tool = cmap({
 667             "inputs": [
 668                 {"id": "p1",
 669                  "type": "Directory"}
 670             ],
 671             "outputs": [],
 672             "baseCommand": "ls",
 673             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 674             "id": "",
 675             "class": "CommandLineTool",
 676             "cwlVersion": "v1.2"
 677         })
 678
 679         loadingContext, runtimeContext = self.helper(runner)
 680         runtimeContext.name = "test_run_mounts"
 681
 682         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 683         arvtool.formatgraph = None
 684         job_order = {
 685             "p1": {
 686                 "class": "Directory",
 687                 "location": "keep:99999999999999999999999999999994+44",
 688                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 689                 "listing": [
 690                     {
 691                         "class": "File",
 692                         "location": "keep:99999999999999999999999999999994+44/file1",
 693                     },
 694                     {
 695                         "class": "File",
 696                         "location": "keep:99999999999999999999999999999994+44/file2",
 697                     }
 698                 ]
 699             }
 700         }
 701         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 702             j.run(runtimeContext)
 703             runner.api.container_requests().create.assert_called_with(
 704                 body=JsonDiffMatcher({
 705                     'environment': {
 706                         'HOME': '/var/spool/cwl',
 707                         'TMPDIR': '/tmp'
 708                     },
 709                     'name': 'test_run_mounts',
 710                     'runtime_constraints': {
 711                         'vcpus': 1,
 712                         'ram': 268435456
 713                     },
 714                     'use_existing': True,
 715                     'priority': 500,
 716                     'mounts': {
 717                         "/keep/99999999999999999999999999999994+44": {
 718                             "kind": "collection",
 719                             "portable_data_hash": "99999999999999999999999999999994+44",
 720                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 721                         },
 722                         '/tmp': {'kind': 'tmp',
 723                                  "capacity": 1073741824 },
 724                         '/var/spool/cwl': {'kind': 'tmp',
 725                                            "capacity": 1073741824 }
 726                     },
 727                     'state': 'Committed',
 728                     'output_name': 'Output from step test_run_mounts',
 729                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 730                     'output_path': '/var/spool/cwl',
 731                     'output_ttl': 0,
 732                     'container_image': '99999999999999999999999999999994+99',
 733                     'command': ['ls', '/var/spool/cwl'],
 734                     'cwd': '/var/spool/cwl',
 735                     'scheduling_parameters': {},
 736                     'properties': {},
 737                     'secret_mounts': {},
 738                     'output_storage_classes': ["default"]
 739                 }))
 740
 741     # The test passes no builder.resources
 742     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 743     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 744     def test_secrets(self, keepdocker):
 745         arvados_cwl.add_arv_hints()
 746         runner = mock.MagicMock()
 747         runner.ignore_docker_for_reuse = False
 748         runner.intermediate_output_ttl = 0
 749         runner.secret_store = cwltool.secrets.SecretStore()
 750         runner.api._rootDesc = {"revision": "20210628"}
 751
 752         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 753         runner.api.collections().get().execute.return_value = {
 754             "portable_data_hash": "99999999999999999999999999999993+99"}
 755
 756         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 757
 758         tool = cmap({"arguments": ["md5sum", "example.conf"],
 759                      "class": "CommandLineTool",
 760                      "cwlVersion": "v1.2",
 761                      "hints": [
 762                          {
 763                              "class": "http://commonwl.org/cwltool#Secrets",
 764                              "secrets": [
 765                                  "#secret_job.cwl/pw"
 766                              ]
 767                          }
 768                      ],
 769                      "id": "",
 770                      "inputs": [
 771                          {
 772                              "id": "#secret_job.cwl/pw",
 773                              "type": "string"
 774                          }
 775                      ],
 776                      "outputs": [
 777                      ],
 778                      "requirements": [
 779                          {
 780                              "class": "InitialWorkDirRequirement",
 781                              "listing": [
 782                                  {
 783                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 784                                      "entryname": "example.conf"
 785                                  }
 786                              ]
 787                          }
 788                      ]})
 789
 790         loadingContext, runtimeContext = self.helper(runner)
 791         runtimeContext.name = "test_secrets"
 792
 793         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 794         arvtool.formatgraph = None
 795
 796         job_order = {"pw": "blorp"}
 797         runner.secret_store.store(["pw"], job_order)
 798
 799         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 800             j.run(runtimeContext)
 801             runner.api.container_requests().create.assert_called_with(
 802                 body=JsonDiffMatcher({
 803                     'environment': {
 804                         'HOME': '/var/spool/cwl',
 805                         'TMPDIR': '/tmp'
 806                     },
 807                     'name': 'test_secrets',
 808                     'runtime_constraints': {
 809                         'vcpus': 1,
 810                         'ram': 268435456
 811                     },
 812                     'use_existing': True,
 813                     'priority': 500,
 814                     'mounts': {
 815                         '/tmp': {'kind': 'tmp',
 816                                  "capacity": 1073741824
 817                              },
 818                         '/var/spool/cwl': {'kind': 'tmp',
 819                                            "capacity": 1073741824 }
 820                     },
 821                     'state': 'Committed',
 822                     'output_name': 'Output from step test_secrets',
 823                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 824                     'output_path': '/var/spool/cwl',
 825                     'output_ttl': 0,
 826                     'container_image': '99999999999999999999999999999993+99',
 827                     'command': ['md5sum', 'example.conf'],
 828                     'cwd': '/var/spool/cwl',
 829                     'scheduling_parameters': {},
 830                     'properties': {},
 831                     "secret_mounts": {
 832                         "/var/spool/cwl/example.conf": {
 833                             "content": "username: user\npassword: blorp\n",
 834                             "kind": "text"
 835                         }
 836                     },
 837                     'output_storage_classes': ["default"]
 838                 }))
 839
 840     # The test passes no builder.resources
 841     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 842     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 843     def test_timelimit(self, keepdocker):
 844         runner = mock.MagicMock()
 845         runner.ignore_docker_for_reuse = False
 846         runner.intermediate_output_ttl = 0
 847         runner.secret_store = cwltool.secrets.SecretStore()
 848         runner.api._rootDesc = {"revision": "20210628"}
 849
 850         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 851         runner.api.collections().get().execute.return_value = {
 852             "portable_data_hash": "99999999999999999999999999999993+99"}
 853
 854         tool = cmap({
 855             "inputs": [],
 856             "outputs": [],
 857             "baseCommand": "ls",
 858             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 859             "id": "",
 860             "cwlVersion": "v1.2",
 861             "class": "CommandLineTool",
 862             "hints": [
 863                 {
 864                     "class": "ToolTimeLimit",
 865                     "timelimit": 42
 866                 }
 867             ]
 868         })
 869
 870         loadingContext, runtimeContext = self.helper(runner)
 871         runtimeContext.name = "test_timelimit"
 872
 873         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 874         arvtool.formatgraph = None
 875
 876         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 877             j.run(runtimeContext)
 878
 879         _, kwargs = runner.api.container_requests().create.call_args
 880         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 881
 882
 883     # The test passes no builder.resources
 884     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 885     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 886     def test_setting_storage_class(self, keepdocker):
 887         arv_docker_clear_cache()
 888
 889         runner = mock.MagicMock()
 890         runner.ignore_docker_for_reuse = False
 891         runner.intermediate_output_ttl = 0
 892         runner.secret_store = cwltool.secrets.SecretStore()
 893         runner.api._rootDesc = {"revision": "20210628"}
 894
 895         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 896         runner.api.collections().get().execute.return_value = {
 897             "portable_data_hash": "99999999999999999999999999999993+99"}
 898
 899         tool = cmap({
 900             "inputs": [],
 901             "outputs": [],
 902             "baseCommand": "ls",
 903             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 904             "id": "",
 905             "cwlVersion": "v1.2",
 906             "class": "CommandLineTool",
 907             "hints": [
 908                 {
 909                     "class": "http://arvados.org/cwl#OutputStorageClass",
 910                     "finalStorageClass": ["baz_sc", "qux_sc"],
 911                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 912                 }
 913             ]
 914         })
 915
 916         loadingContext, runtimeContext = self.helper(runner, True)
 917
 918         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 919         arvtool.formatgraph = None
 920
 921         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 922             j.run(runtimeContext)
 923             runner.api.container_requests().create.assert_called_with(
 924                 body=JsonDiffMatcher({
 925                     'environment': {
 926                         'HOME': '/var/spool/cwl',
 927                         'TMPDIR': '/tmp'
 928                     },
 929                     'name': 'test_run_True',
 930                     'runtime_constraints': {
 931                         'vcpus': 1,
 932                         'ram': 268435456
 933                     },
 934                     'use_existing': True,
 935                     'priority': 500,
 936                     'mounts': {
 937                         '/tmp': {'kind': 'tmp',
 938                                  "capacity": 1073741824
 939                              },
 940                         '/var/spool/cwl': {'kind': 'tmp',
 941                                            "capacity": 1073741824 }
 942                     },
 943                     'state': 'Committed',
 944                     'output_name': 'Output from step test_run_True',
 945                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 946                     'output_path': '/var/spool/cwl',
 947                     'output_ttl': 0,
 948                     'container_image': '99999999999999999999999999999993+99',
 949                     'command': ['ls', '/var/spool/cwl'],
 950                     'cwd': '/var/spool/cwl',
 951                     'scheduling_parameters': {},
 952                     'properties': {},
 953                     'secret_mounts': {},
 954                     'output_storage_classes': ["foo_sc", "bar_sc"]
 955                 }))
 956
 957
 958     # The test passes no builder.resources
 959     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 960     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 961     def test_setting_process_properties(self, keepdocker):
 962         arv_docker_clear_cache()
 963
 964         runner = mock.MagicMock()
 965         runner.ignore_docker_for_reuse = False
 966         runner.intermediate_output_ttl = 0
 967         runner.secret_store = cwltool.secrets.SecretStore()
 968         runner.api._rootDesc = {"revision": "20210628"}
 969
 970         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 971         runner.api.collections().get().execute.return_value = {
 972             "portable_data_hash": "99999999999999999999999999999993+99"}
 973
 974         tool = cmap({
 975             "inputs": [
 976                 {"id": "x", "type": "string"}],
 977             "outputs": [],
 978             "baseCommand": "ls",
 979             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 980             "id": "",
 981             "class": "CommandLineTool",
 982             "cwlVersion": "v1.2",
 983             "hints": [
 984             {
 985                 "class": "http://arvados.org/cwl#ProcessProperties",
 986                 "processProperties": [
 987                     {"propertyName": "foo",
 988                      "propertyValue": "bar"},
 989                     {"propertyName": "baz",
 990                      "propertyValue": "$(inputs.x)"},
 991                     {"propertyName": "quux",
 992                      "propertyValue": {
 993                          "q1": 1,
 994                          "q2": 2
 995                      }
 996                     }
 997                 ],
 998             }
 999         ]
1000         })
1001
1002         loadingContext, runtimeContext = self.helper(runner, True)
1003
1004         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1005         arvtool.formatgraph = None
1006
1007         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1008             j.run(runtimeContext)
1009             runner.api.container_requests().create.assert_called_with(
1010                 body=JsonDiffMatcher({
1011                     'environment': {
1012                         'HOME': '/var/spool/cwl',
1013                         'TMPDIR': '/tmp'
1014                     },
1015                     'name': 'test_run_True',
1016                     'runtime_constraints': {
1017                         'vcpus': 1,
1018                         'ram': 268435456
1019                     },
1020                     'use_existing': True,
1021                     'priority': 500,
1022                     'mounts': {
1023                         '/tmp': {'kind': 'tmp',
1024                                  "capacity": 1073741824
1025                              },
1026                         '/var/spool/cwl': {'kind': 'tmp',
1027                                            "capacity": 1073741824 }
1028                     },
1029                     'state': 'Committed',
1030                     'output_name': 'Output from step test_run_True',
1031                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1032                     'output_path': '/var/spool/cwl',
1033                     'output_ttl': 0,
1034                     'container_image': '99999999999999999999999999999993+99',
1035                     'command': ['ls', '/var/spool/cwl'],
1036                     'cwd': '/var/spool/cwl',
1037                     'scheduling_parameters': {},
1038                     'properties': {
1039                         "baz": "blorp",
1040                         "foo": "bar",
1041                         "quux": {
1042                             "q1": 1,
1043                             "q2": 2
1044                         }
1045                     },
1046                     'secret_mounts': {},
1047                     'output_storage_classes': ["default"]
1048                 }))
1049
1050
1051     # The test passes no builder.resources
1052     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1053     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1054     def test_cuda_requirement(self, keepdocker):
1055         arvados_cwl.add_arv_hints()
1056         arv_docker_clear_cache()
1057
1058         runner = mock.MagicMock()
1059         runner.ignore_docker_for_reuse = False
1060         runner.intermediate_output_ttl = 0
1061         runner.secret_store = cwltool.secrets.SecretStore()
1062         runner.api._rootDesc = {"revision": "20210628"}
1063
1064         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1065         runner.api.collections().get().execute.return_value = {
1066             "portable_data_hash": "99999999999999999999999999999993+99"}
1067
1068         test_cwl_req = [{
1069                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1070                 "cudaVersionMin": "11.0",
1071                 "cudaComputeCapability": "9.0",
1072             }, {
1073                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1074                 "cudaVersionMin": "11.0",
1075                 "cudaComputeCapability": "9.0",
1076                 "cudaDeviceCountMin": 2
1077             }, {
1078                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1079                 "cudaVersionMin": "11.0",
1080                 "cudaComputeCapability": ["4.0", "5.0"],
1081                 "cudaDeviceCountMin": 2
1082             }]
1083
1084         test_arv_req = [{
1085             'device_count': 1,
1086             'driver_version': "11.0",
1087             'hardware_capability': "9.0"
1088         }, {
1089             'device_count': 2,
1090             'driver_version': "11.0",
1091             'hardware_capability': "9.0"
1092         }, {
1093             'device_count': 2,
1094             'driver_version': "11.0",
1095             'hardware_capability': "4.0"
1096         }]
1097
1098         for test_case in range(0, len(test_cwl_req)):
1099
1100             tool = cmap({
1101                 "inputs": [],
1102                 "outputs": [],
1103                 "baseCommand": "nvidia-smi",
1104                 "arguments": [],
1105                 "id": "",
1106                 "cwlVersion": "v1.2",
1107                 "class": "CommandLineTool",
1108                 "requirements": [test_cwl_req[test_case]]
1109             })
1110
1111             loadingContext, runtimeContext = self.helper(runner, True)
1112
1113             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1114             arvtool.formatgraph = None
1115
1116             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1117                 j.run(runtimeContext)
1118                 runner.api.container_requests().create.assert_called_with(
1119                     body=JsonDiffMatcher({
1120                         'environment': {
1121                             'HOME': '/var/spool/cwl',
1122                             'TMPDIR': '/tmp'
1123                         },
1124                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1125                         'runtime_constraints': {
1126                             'vcpus': 1,
1127                             'ram': 268435456,
1128                             'cuda': test_arv_req[test_case]
1129                         },
1130                         'use_existing': True,
1131                         'priority': 500,
1132                         'mounts': {
1133                             '/tmp': {'kind': 'tmp',
1134                                      "capacity": 1073741824
1135                                  },
1136                             '/var/spool/cwl': {'kind': 'tmp',
1137                                                "capacity": 1073741824 }
1138                         },
1139                         'state': 'Committed',
1140                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1141                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1142                         'output_path': '/var/spool/cwl',
1143                         'output_ttl': 0,
1144                         'container_image': '99999999999999999999999999999993+99',
1145                         'command': ['nvidia-smi'],
1146                         'cwd': '/var/spool/cwl',
1147                         'scheduling_parameters': {},
1148                         'properties': {},
1149                         'secret_mounts': {},
1150                         'output_storage_classes': ["default"]
1151                     }))
1152
1153
1154     # The test passes no builder.resources
1155     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1156     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1157     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1158     def test_match_local_docker(self, keepdocker, determine_image_id):
1159         arvados_cwl.add_arv_hints()
1160         arv_docker_clear_cache()
1161
1162         runner = mock.MagicMock()
1163         runner.ignore_docker_for_reuse = False
1164         runner.intermediate_output_ttl = 0
1165         runner.secret_store = cwltool.secrets.SecretStore()
1166         runner.api._rootDesc = {"revision": "20210628"}
1167
1168         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1169                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1170         determine_image_id.side_effect = lambda x: "123"
1171         def execute(uuid):
1172             ex = mock.MagicMock()
1173             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1174                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1175             ex.execute.return_value = lookup[uuid]
1176             return ex
1177         runner.api.collections().get.side_effect = execute
1178
1179         tool = cmap({
1180             "inputs": [],
1181             "outputs": [],
1182             "baseCommand": "echo",
1183             "arguments": [],
1184             "id": "",
1185             "cwlVersion": "v1.0",
1186             "class": "org.w3id.cwl.cwl.CommandLineTool"
1187         })
1188
1189         loadingContext, runtimeContext = self.helper(runner, True)
1190
1191         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1192         arvtool.formatgraph = None
1193
1194         container_request = {
1195             'environment': {
1196                 'HOME': '/var/spool/cwl',
1197                 'TMPDIR': '/tmp'
1198             },
1199             'name': 'test_run_True',
1200             'runtime_constraints': {
1201                 'vcpus': 1,
1202                 'ram': 1073741824,
1203             },
1204             'use_existing': True,
1205             'priority': 500,
1206             'mounts': {
1207                 '/tmp': {'kind': 'tmp',
1208                          "capacity": 1073741824
1209                          },
1210                 '/var/spool/cwl': {'kind': 'tmp',
1211                                    "capacity": 1073741824 }
1212             },
1213             'state': 'Committed',
1214             'output_name': 'Output from step test_run_True',
1215             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1216             'output_path': '/var/spool/cwl',
1217             'output_ttl': 0,
1218             'container_image': '99999999999999999999999999999994+99',
1219             'command': ['echo'],
1220             'cwd': '/var/spool/cwl',
1221             'scheduling_parameters': {},
1222             'properties': {},
1223             'secret_mounts': {},
1224             'output_storage_classes': ["default"]
1225         }
1226
1227         runtimeContext.match_local_docker = False
1228         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1229             j.run(runtimeContext)
1230             runner.api.container_requests().create.assert_called_with(
1231                 body=JsonDiffMatcher(container_request))
1232
1233         arv_docker_clear_cache()
1234         runtimeContext.match_local_docker = True
1235         container_request['container_image'] = '99999999999999999999999999999993+99'
1236         container_request['name'] = 'test_run_True_2'
1237         container_request['output_name'] = 'Output from step test_run_True_2'
1238         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1239             j.run(runtimeContext)
1240             runner.api.container_requests().create.assert_called_with(
1241                 body=JsonDiffMatcher(container_request))
1242
1243
1244     # The test passes no builder.resources
1245     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1246     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1247     def test_run_preemptible_hint(self, keepdocker):
1248         arvados_cwl.add_arv_hints()
1249         for enable_preemptible in (None, True, False):
1250             for preemptible_hint in (None, True, False):
1251                 arv_docker_clear_cache()
1252
1253                 runner = mock.MagicMock()
1254                 runner.ignore_docker_for_reuse = False
1255                 runner.intermediate_output_ttl = 0
1256                 runner.secret_store = cwltool.secrets.SecretStore()
1257                 runner.api._rootDesc = {"revision": "20210628"}
1258
1259                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1260                 runner.api.collections().get().execute.return_value = {
1261                     "portable_data_hash": "99999999999999999999999999999993+99"}
1262
1263                 if preemptible_hint is not None:
1264                     hints = [{
1265                         "class": "http://arvados.org/cwl#UsePreemptible",
1266                         "usePreemptible": preemptible_hint
1267                     }]
1268                 else:
1269                     hints = []
1270
1271                 tool = cmap({
1272                     "inputs": [],
1273                     "outputs": [],
1274                     "baseCommand": "ls",
1275                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1276                     "id": "",
1277                     "class": "CommandLineTool",
1278                     "cwlVersion": "v1.2",
1279                     "hints": hints
1280                 })
1281
1282                 loadingContext, runtimeContext = self.helper(runner)
1283
1284                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1285                 runtimeContext.enable_preemptible = enable_preemptible
1286
1287                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1288                 arvtool.formatgraph = None
1289
1290                 # Test the interactions between --enable/disable-preemptible
1291                 # and UsePreemptible hint
1292
1293                 if enable_preemptible is None:
1294                     if preemptible_hint is None:
1295                         sched = {}
1296                     else:
1297                         sched = {'preemptible': preemptible_hint}
1298                 else:
1299                     if preemptible_hint is None:
1300                         sched = {'preemptible': enable_preemptible}
1301                     else:
1302                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1303
1304                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1305                     j.run(runtimeContext)
1306                     runner.api.container_requests().create.assert_called_with(
1307                         body=JsonDiffMatcher({
1308                             'environment': {
1309                                 'HOME': '/var/spool/cwl',
1310                                 'TMPDIR': '/tmp'
1311                             },
1312                             'name': runtimeContext.name,
1313                             'runtime_constraints': {
1314                                 'vcpus': 1,
1315                                 'ram': 268435456
1316                             },
1317                             'use_existing': True,
1318                             'priority': 500,
1319                             'mounts': {
1320                                 '/tmp': {'kind': 'tmp',
1321                                          "capacity": 1073741824
1322                                      },
1323                                 '/var/spool/cwl': {'kind': 'tmp',
1324                                                    "capacity": 1073741824 }
1325                             },
1326                             'state': 'Committed',
1327                             'output_name': 'Output from step '+runtimeContext.name,
1328                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1329                             'output_path': '/var/spool/cwl',
1330                             'output_ttl': 0,
1331                             'container_image': '99999999999999999999999999999993+99',
1332                             'command': ['ls', '/var/spool/cwl'],
1333                             'cwd': '/var/spool/cwl',
1334                             'scheduling_parameters': sched,
1335                             'properties': {},
1336                             'secret_mounts': {},
1337                             'output_storage_classes': ["default"]
1338                         }))
1339
1340
1341     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1342     def test_output_properties(self, keepdocker):
1343         arvados_cwl.add_arv_hints()
1344         for rev in ["20210628", "20220510"]:
1345             runner = mock.MagicMock()
1346             runner.ignore_docker_for_reuse = False
1347             runner.intermediate_output_ttl = 0
1348             runner.secret_store = cwltool.secrets.SecretStore()
1349             runner.api._rootDesc = {"revision": rev}
1350
1351             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1352             runner.api.collections().get().execute.return_value = {
1353                 "portable_data_hash": "99999999999999999999999999999993+99"}
1354
1355             tool = cmap({
1356                 "inputs": [{
1357                     "id": "inp",
1358                     "type": "string"
1359                 }],
1360                 "outputs": [],
1361                 "baseCommand": "ls",
1362                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1363                 "id": "",
1364                 "cwlVersion": "v1.2",
1365                 "class": "CommandLineTool",
1366                 "hints": [
1367                     {
1368                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1369                         "outputProperties": {
1370                             "foo": "bar",
1371                             "baz": "$(inputs.inp)"
1372                         }
1373                     }
1374                 ]
1375             })
1376
1377             loadingContext, runtimeContext = self.helper(runner)
1378             runtimeContext.name = "test_timelimit"
1379
1380             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1381             arvtool.formatgraph = None
1382
1383             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1384                 j.run(runtimeContext)
1385
1386             _, kwargs = runner.api.container_requests().create.call_args
1387             if rev == "20220510":
1388                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1389             else:
1390                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1391
1392
1393 class TestWorkflow(unittest.TestCase):
1394     def setUp(self):
1395         cwltool.process._names = set()
1396         arv_docker_clear_cache()
1397
1398     def helper(self, runner, enable_reuse=True):
1399         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1400
1401         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1402                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1403
1404         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1405         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1406         document_loader.fetch_text = document_loader.fetcher.fetch_text
1407         document_loader.check_exists = document_loader.fetcher.check_exists
1408
1409         loadingContext = arvados_cwl.context.ArvLoadingContext(
1410             {"avsc_names": avsc_names,
1411              "basedir": "",
1412              "make_fs_access": make_fs_access,
1413              "loader": document_loader,
1414              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1415              "construct_tool_object": runner.arv_make_tool})
1416         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1417             {"work_api": "containers",
1418              "basedir": "",
1419              "name": "test_run_wf_"+str(enable_reuse),
1420              "make_fs_access": make_fs_access,
1421              "tmpdir": "/tmp",
1422              "enable_reuse": enable_reuse,
1423              "priority": 500})
1424
1425         return loadingContext, runtimeContext
1426
1427     # The test passes no builder.resources
1428     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1429     @mock.patch("arvados.collection.CollectionReader")
1430     @mock.patch("arvados.collection.Collection")
1431     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1432     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1433         arvados_cwl.add_arv_hints()
1434
1435         api = mock.MagicMock()
1436         api._rootDesc = get_rootDesc()
1437
1438         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1439         self.assertEqual(runner.work_api, 'containers')
1440
1441         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1442         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1443         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1444                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1445
1446         runner.api.containers().current().execute.return_value = {}
1447
1448         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1449         runner.ignore_docker_for_reuse = False
1450         runner.num_retries = 0
1451         runner.secret_store = cwltool.secrets.SecretStore()
1452
1453         loadingContext, runtimeContext = self.helper(runner)
1454         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1455
1456         mockcollectionreader().exists.return_value = True
1457
1458         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1459         metadata["cwlVersion"] = tool["cwlVersion"]
1460
1461         mockc = mock.MagicMock()
1462         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1463         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1464
1465         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1466         arvtool.formatgraph = None
1467         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1468
1469         next(it).run(runtimeContext)
1470         next(it).run(runtimeContext)
1471
1472         with open("tests/wf/scatter2_subwf.cwl") as f:
1473             subwf = StripYAMLComments(f.read()).rstrip()
1474
1475         runner.api.container_requests().create.assert_called_with(
1476             body=JsonDiffMatcher({
1477                 "command": [
1478                     "cwltool",
1479                     "--no-container",
1480                     "--move-outputs",
1481                     "--preserve-entire-environment",
1482                     "workflow.cwl",
1483                     "cwl.input.yml"
1484                 ],
1485                 "container_image": "99999999999999999999999999999993+99",
1486                 "cwd": "/var/spool/cwl",
1487                 "environment": {
1488                     "HOME": "/var/spool/cwl",
1489                     "TMPDIR": "/tmp"
1490                 },
1491                 "mounts": {
1492                     "/keep/99999999999999999999999999999999+118": {
1493                         "kind": "collection",
1494                         "portable_data_hash": "99999999999999999999999999999999+118"
1495                     },
1496                     "/tmp": {
1497                         "capacity": 1073741824,
1498                         "kind": "tmp"
1499                     },
1500                     "/var/spool/cwl": {
1501                         "capacity": 1073741824,
1502                         "kind": "tmp"
1503                     },
1504                     "/var/spool/cwl/cwl.input.yml": {
1505                         "kind": "collection",
1506                         "path": "cwl.input.yml",
1507                         "portable_data_hash": "99999999999999999999999999999996+99"
1508                     },
1509                     "/var/spool/cwl/workflow.cwl": {
1510                         "kind": "collection",
1511                         "path": "workflow.cwl",
1512                         "portable_data_hash": "99999999999999999999999999999996+99"
1513                     },
1514                     "stdout": {
1515                         "kind": "file",
1516                         "path": "/var/spool/cwl/cwl.output.json"
1517                     }
1518                 },
1519                 "name": "scatterstep",
1520                 "output_name": "Output from step scatterstep",
1521                 "output_path": "/var/spool/cwl",
1522                 "output_ttl": 0,
1523                 "priority": 500,
1524                 "properties": {},
1525                 "runtime_constraints": {
1526                     "ram": 1073741824,
1527                     "vcpus": 1
1528                 },
1529                 "scheduling_parameters": {},
1530                 "secret_mounts": {},
1531                 "state": "Committed",
1532                 "use_existing": True,
1533                 'output_storage_classes': ["default"]
1534             }))
1535         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1536         mockc.open().__enter__().write.assert_has_calls([mock.call(
1537 '''{
1538   "fileblub": {
1539     "basename": "token.txt",
1540     "class": "File",
1541     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1542     "size": 0
1543   },
1544   "sleeptime": 5
1545 }''')])
1546
1547     # The test passes no builder.resources
1548     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1549     @mock.patch("arvados.collection.CollectionReader")
1550     @mock.patch("arvados.collection.Collection")
1551     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1552     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1553         arvados_cwl.add_arv_hints()
1554
1555         api = mock.MagicMock()
1556         api._rootDesc = get_rootDesc()
1557
1558         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1559         self.assertEqual(runner.work_api, 'containers')
1560
1561         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1562         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1563                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1564         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1565                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1566
1567         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1568         runner.ignore_docker_for_reuse = False
1569         runner.num_retries = 0
1570         runner.secret_store = cwltool.secrets.SecretStore()
1571
1572         loadingContext, runtimeContext = self.helper(runner)
1573         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1574         loadingContext.do_update = True
1575         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1576
1577         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1578
1579         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1580         arvtool.formatgraph = None
1581         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1582
1583         next(it).run(runtimeContext)
1584         next(it).run(runtimeContext)
1585
1586         with open("tests/wf/echo-subwf.cwl") as f:
1587             subwf = StripYAMLComments(f.read())
1588
1589         runner.api.container_requests().create.assert_called_with(
1590             body=JsonDiffMatcher({
1591                 'output_ttl': 0,
1592                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1593                 'scheduling_parameters': {},
1594                 'name': u'echo-subwf',
1595                 'secret_mounts': {},
1596                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1597                 'properties': {},
1598                 'priority': 500,
1599                 'mounts': {
1600                     '/var/spool/cwl/cwl.input.yml': {
1601                         'portable_data_hash': '99999999999999999999999999999996+99',
1602                         'kind': 'collection',
1603                         'path': 'cwl.input.yml'
1604                     },
1605                     '/var/spool/cwl/workflow.cwl': {
1606                         'portable_data_hash': '99999999999999999999999999999996+99',
1607                         'kind': 'collection',
1608                         'path': 'workflow.cwl'
1609                     },
1610                     'stdout': {
1611                         'path': '/var/spool/cwl/cwl.output.json',
1612                         'kind': 'file'
1613                     },
1614                     '/tmp': {
1615                         'kind': 'tmp',
1616                         'capacity': 1073741824
1617                     }, '/var/spool/cwl': {
1618                         'kind': 'tmp',
1619                         'capacity': 3221225472
1620                     }
1621                 },
1622                 'state': 'Committed',
1623                 'output_path': '/var/spool/cwl',
1624                 'container_image': '99999999999999999999999999999993+99',
1625                 'command': [
1626                     u'cwltool',
1627                     u'--no-container',
1628                     u'--move-outputs',
1629                     u'--preserve-entire-environment',
1630                     u'workflow.cwl',
1631                     u'cwl.input.yml'
1632                 ],
1633                 'use_existing': True,
1634                 'output_name': u'Output from step echo-subwf',
1635                 'cwd': '/var/spool/cwl',
1636                 'output_storage_classes': ["default"]
1637             }))
1638
1639     def test_default_work_api(self):
1640         arvados_cwl.add_arv_hints()
1641
1642         api = mock.MagicMock()
1643         api._rootDesc = copy.deepcopy(get_rootDesc())
1644         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1645         self.assertEqual(runner.work_api, 'containers')