sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import threading
  20 import cwltool.process
  21 import cwltool.secrets
  22 import cwltool.load_tool
  23 from cwltool.update import INTERNAL_VERSION
  24 from schema_salad.ref_resolver import Loader
  25 from schema_salad.sourceline import cmap
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138
 139             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 140             runner.api.collections().get().execute.return_value = {
 141                 "portable_data_hash": "99999999999999999999999999999993+99"}
 142
 143             tool = cmap({
 144                 "inputs": [],
 145                 "outputs": [],
 146                 "baseCommand": "ls",
 147                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 148                 "id": "",
 149                 "class": "CommandLineTool",
 150                 "cwlVersion": "v1.2"
 151             })
 152
 153             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 154
 155             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 156             arvtool.formatgraph = None
 157
 158             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 159                 j.run(runtimeContext)
 160                 runner.api.container_requests().create.assert_called_with(
 161                     body=JsonDiffMatcher({
 162                         'environment': {
 163                             'HOME': '/var/spool/cwl',
 164                             'TMPDIR': '/tmp'
 165                         },
 166                         'name': 'test_run_'+str(enable_reuse),
 167                         'runtime_constraints': {
 168                             'vcpus': 1,
 169                             'ram': 268435456
 170                         },
 171                         'use_existing': enable_reuse,
 172                         'priority': 500,
 173                         'mounts': {
 174                             '/tmp': {'kind': 'tmp',
 175                                      "capacity": 1073741824
 176                                  },
 177                             '/var/spool/cwl': {'kind': 'tmp',
 178                                                "capacity": 1073741824 }
 179                         },
 180                         'state': 'Committed',
 181                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 182                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 183                         'output_path': '/var/spool/cwl',
 184                         'output_ttl': 0,
 185                         'container_image': '99999999999999999999999999999993+99',
 186                         'command': ['ls', '/var/spool/cwl'],
 187                         'cwd': '/var/spool/cwl',
 188                         'scheduling_parameters': {},
 189                         'properties': {},
 190                         'secret_mounts': {},
 191                         'output_storage_classes': ["default"]
 192                     }))
 193
 194     # The test passes some fields in builder.resources
 195     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 196     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 197     def test_resource_requirements(self, keepdocker):
 198         arvados_cwl.add_arv_hints()
 199         runner = mock.MagicMock()
 200         runner.ignore_docker_for_reuse = False
 201         runner.intermediate_output_ttl = 3600
 202         runner.secret_store = cwltool.secrets.SecretStore()
 203         runner.api._rootDesc = {"revision": "20210628"}
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         tool = cmap({
 210             "inputs": [],
 211             "outputs": [],
 212             "hints": [{
 213                 "class": "ResourceRequirement",
 214                 "coresMin": 3,
 215                 "ramMin": 3000,
 216                 "tmpdirMin": 4000,
 217                 "outdirMin": 5000
 218             }, {
 219                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 220                 "keep_cache": 512
 221             }, {
 222                 "class": "http://arvados.org/cwl#APIRequirement",
 223             }, {
 224                 "class": "http://arvados.org/cwl#PartitionRequirement",
 225                 "partition": "blurb"
 226             }, {
 227                 "class": "http://arvados.org/cwl#IntermediateOutput",
 228                 "outputTTL": 7200
 229             }, {
 230                 "class": "WorkReuse",
 231                 "enableReuse": False
 232             }],
 233             "baseCommand": "ls",
 234             "id": "",
 235             "class": "CommandLineTool",
 236             "cwlVersion": "v1.2"
 237         })
 238
 239         loadingContext, runtimeContext = self.helper(runner)
 240         runtimeContext.name = "test_resource_requirements"
 241
 242         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 243         arvtool.formatgraph = None
 244         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 245             j.run(runtimeContext)
 246
 247         call_args, call_kwargs = runner.api.container_requests().create.call_args
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_resource_requirements',
 255             'runtime_constraints': {
 256                 'vcpus': 3,
 257                 'ram': 3145728000,
 258                 'keep_cache_ram': 536870912,
 259                 'API': True
 260             },
 261             'use_existing': False,
 262             'priority': 500,
 263             'mounts': {
 264                 '/tmp': {'kind': 'tmp',
 265                          "capacity": 4194304000 },
 266                 '/var/spool/cwl': {'kind': 'tmp',
 267                                    "capacity": 5242880000 }
 268             },
 269             'state': 'Committed',
 270             'output_name': 'Output for step test_resource_requirements',
 271             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 272             'output_path': '/var/spool/cwl',
 273             'output_ttl': 7200,
 274             'container_image': '99999999999999999999999999999993+99',
 275             'command': ['ls'],
 276             'cwd': '/var/spool/cwl',
 277             'scheduling_parameters': {
 278                 'partitions': ['blurb']
 279             },
 280             'properties': {},
 281             'secret_mounts': {},
 282             'output_storage_classes': ["default"]
 283         }
 284
 285         call_body = call_kwargs.get('body', None)
 286         self.assertNotEqual(None, call_body)
 287         for key in call_body:
 288             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 289
 290
 291     # The test passes some fields in builder.resources
 292     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 293     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 294     @mock.patch("arvados.collection.Collection")
 295     def test_initial_work_dir(self, collection_mock, keepdocker):
 296         runner = mock.MagicMock()
 297         runner.ignore_docker_for_reuse = False
 298         runner.intermediate_output_ttl = 0
 299         runner.secret_store = cwltool.secrets.SecretStore()
 300         runner.api._rootDesc = {"revision": "20210628"}
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         sourcemock = mock.MagicMock()
 307         def get_collection_mock(p):
 308             if "/" in p:
 309                 return (sourcemock, p.split("/", 1)[1])
 310             else:
 311                 return (sourcemock, "")
 312         runner.fs_access.get_collection.side_effect = get_collection_mock
 313
 314         vwdmock = mock.MagicMock()
 315         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 316
 317         tool = cmap({
 318             "inputs": [],
 319             "outputs": [],
 320             "hints": [{
 321                 "class": "InitialWorkDirRequirement",
 322                 "listing": [{
 323                     "class": "File",
 324                     "basename": "foo",
 325                     "location": "keep:99999999999999999999999999999995+99/bar"
 326                 },
 327                 {
 328                     "class": "Directory",
 329                     "basename": "foo2",
 330                     "location": "keep:99999999999999999999999999999995+99"
 331                 },
 332                 {
 333                     "class": "File",
 334                     "basename": "filename",
 335                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 336                 },
 337                 {
 338                     "class": "Directory",
 339                     "basename": "subdir",
 340                     "location": "keep:99999999999999999999999999999995+99/subdir"
 341                 }                        ]
 342             }],
 343             "baseCommand": "ls",
 344             "class": "CommandLineTool",
 345             "cwlVersion": "v1.2",
 346             "id": ""
 347         })
 348
 349         loadingContext, runtimeContext = self.helper(runner)
 350         runtimeContext.name = "test_initial_work_dir"
 351
 352         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 353
 354         arvtool.formatgraph = None
 355         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 356             j.run(runtimeContext)
 357
 358         call_args, call_kwargs = runner.api.container_requests().create.call_args
 359
 360         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 361         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 362         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 363         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 364
 365         call_body_expected = {
 366             'environment': {
 367                 'HOME': '/var/spool/cwl',
 368                 'TMPDIR': '/tmp'
 369             },
 370             'name': 'test_initial_work_dir',
 371             'runtime_constraints': {
 372                 'vcpus': 1,
 373                 'ram': 268435456
 374             },
 375             'use_existing': True,
 376             'priority': 500,
 377             'mounts': {
 378                 '/tmp': {'kind': 'tmp',
 379                          "capacity": 1073741824 },
 380                 '/var/spool/cwl': {'kind': 'tmp',
 381                                    "capacity": 1073741824 },
 382                 '/var/spool/cwl/foo': {
 383                     'kind': 'collection',
 384                     'path': 'foo',
 385                     'portable_data_hash': '99999999999999999999999999999996+99'
 386                 },
 387                 '/var/spool/cwl/foo2': {
 388                     'kind': 'collection',
 389                     'path': 'foo2',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/filename': {
 393                     'kind': 'collection',
 394                     'path': 'filename',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/subdir': {
 398                     'kind': 'collection',
 399                     'path': 'subdir',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 }
 402             },
 403             'state': 'Committed',
 404             'output_name': 'Output for step test_initial_work_dir',
 405             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 406             'output_path': '/var/spool/cwl',
 407             'output_ttl': 0,
 408             'container_image': '99999999999999999999999999999993+99',
 409             'command': ['ls'],
 410             'cwd': '/var/spool/cwl',
 411             'scheduling_parameters': {
 412             },
 413             'properties': {},
 414             'secret_mounts': {},
 415             'output_storage_classes': ["default"]
 416         }
 417
 418         call_body = call_kwargs.get('body', None)
 419         self.assertNotEqual(None, call_body)
 420         for key in call_body:
 421             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 422
 423
 424     # Test redirecting stdin/stdout/stderr
 425     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 426     def test_redirects(self, keepdocker):
 427         runner = mock.MagicMock()
 428         runner.ignore_docker_for_reuse = False
 429         runner.intermediate_output_ttl = 0
 430         runner.secret_store = cwltool.secrets.SecretStore()
 431         runner.api._rootDesc = {"revision": "20210628"}
 432
 433         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 434         runner.api.collections().get().execute.return_value = {
 435             "portable_data_hash": "99999999999999999999999999999993+99"}
 436
 437         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 438
 439         tool = cmap({
 440             "inputs": [],
 441             "outputs": [],
 442             "baseCommand": "ls",
 443             "stdout": "stdout.txt",
 444             "stderr": "stderr.txt",
 445             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 446             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 447             "id": "",
 448             "class": "CommandLineTool",
 449             "cwlVersion": "v1.2"
 450         })
 451
 452         loadingContext, runtimeContext = self.helper(runner)
 453         runtimeContext.name = "test_run_redirect"
 454
 455         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 456         arvtool.formatgraph = None
 457         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 458             j.run(runtimeContext)
 459             runner.api.container_requests().create.assert_called_with(
 460                 body=JsonDiffMatcher({
 461                     'environment': {
 462                         'HOME': '/var/spool/cwl',
 463                         'TMPDIR': '/tmp'
 464                     },
 465                     'name': 'test_run_redirect',
 466                     'runtime_constraints': {
 467                         'vcpus': 1,
 468                         'ram': 268435456
 469                     },
 470                     'use_existing': True,
 471                     'priority': 500,
 472                     'mounts': {
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 },
 477                         "stderr": {
 478                             "kind": "file",
 479                             "path": "/var/spool/cwl/stderr.txt"
 480                         },
 481                         "stdin": {
 482                             "kind": "collection",
 483                             "path": "file.txt",
 484                             "portable_data_hash": "99999999999999999999999999999996+99"
 485                         },
 486                         "stdout": {
 487                             "kind": "file",
 488                             "path": "/var/spool/cwl/stdout.txt"
 489                         },
 490                     },
 491                     'state': 'Committed',
 492                     "output_name": "Output for step test_run_redirect",
 493                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 494                     'output_path': '/var/spool/cwl',
 495                     'output_ttl': 0,
 496                     'container_image': '99999999999999999999999999999993+99',
 497                     'command': ['ls', '/var/spool/cwl'],
 498                     'cwd': '/var/spool/cwl',
 499                     'scheduling_parameters': {},
 500                     'properties': {},
 501                     'secret_mounts': {},
 502                     'output_storage_classes': ["default"]
 503                 }))
 504
 505     @mock.patch("arvados.collection.Collection")
 506     def test_done(self, col):
 507         api = mock.MagicMock()
 508
 509         runner = mock.MagicMock()
 510         runner.api = api
 511         runner.num_retries = 0
 512         runner.ignore_docker_for_reuse = False
 513         runner.intermediate_output_ttl = 0
 514         runner.secret_store = cwltool.secrets.SecretStore()
 515
 516         runner.api.containers().get().execute.return_value = {"state":"Complete",
 517                                                               "output": "abc+123",
 518                                                               "exit_code": 0}
 519
 520         col().open.return_value = []
 521
 522         loadingContext, runtimeContext = self.helper(runner)
 523
 524         arvjob = arvados_cwl.ArvadosContainer(runner,
 525                                               runtimeContext,
 526                                               mock.MagicMock(),
 527                                               {},
 528                                               None,
 529                                               [],
 530                                               [],
 531                                               "testjob")
 532         arvjob.output_callback = mock.MagicMock()
 533         arvjob.collect_outputs = mock.MagicMock()
 534         arvjob.successCodes = [0]
 535         arvjob.outdir = "/var/spool/cwl"
 536         arvjob.output_ttl = 3600
 537
 538         arvjob.collect_outputs.return_value = {"out": "stuff"}
 539
 540         arvjob.done({
 541             "state": "Final",
 542             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 543             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 544             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 545             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 546             "modified_at": "2017-05-26T12:01:22Z"
 547         })
 548
 549         self.assertFalse(api.collections().create.called)
 550         self.assertFalse(runner.runtime_status_error.called)
 551
 552         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 553         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 554         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 555
 556     # Test to make sure we dont call runtime_status_update if we already did
 557     # some where higher up in the call stack
 558     @mock.patch("arvados_cwl.util.get_current_container")
 559     def test_recursive_runtime_status_update(self, gcc_mock):
 560         self.setup_and_test_container_executor_and_logging(gcc_mock)
 561         root_logger = logging.getLogger('')
 562
 563         # get_current_container is invoked when we call runtime_status_update
 564         # so try and log again!
 565         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 566         try:
 567             root_logger.error("First Error")
 568         except RuntimeError:
 569             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 570
 571
 572     # Test to make sure that an exception raised from
 573     # get_current_container doesn't cause the logger to raise an
 574     # exception
 575     @mock.patch("arvados_cwl.util.get_current_container")
 576     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 577         self.setup_and_test_container_executor_and_logging(gcc_mock)
 578         root_logger = logging.getLogger('')
 579
 580         # get_current_container is invoked when we call
 581         # runtime_status_update, it is going to also raise an
 582         # exception.
 583         gcc_mock.side_effect = Exception("Second Error")
 584         try:
 585             root_logger.error("First Error")
 586         except Exception:
 587             self.fail("Exception in logger should not propagate")
 588         self.assertTrue(gcc_mock.called)
 589
 590     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 591     @mock.patch("arvados_cwl.util.get_current_container")
 592     @mock.patch("arvados.collection.CollectionReader")
 593     @mock.patch("arvados.collection.Collection")
 594     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 595         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 596
 597         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 598         self.assertTrue(gcc_mock.called)
 599
 600         runner.num_retries = 0
 601         runner.ignore_docker_for_reuse = False
 602         runner.intermediate_output_ttl = 0
 603         runner.secret_store = cwltool.secrets.SecretStore()
 604         runner.label = mock.MagicMock()
 605         runner.label.return_value = '[container testjob]'
 606
 607         runner.api.containers().get().execute.return_value = {
 608             "state":"Complete",
 609             "output": "abc+123",
 610             "exit_code": 1,
 611             "log": "def+234"
 612         }
 613
 614         col().open.return_value = []
 615
 616         loadingContext, runtimeContext = self.helper(runner)
 617
 618         arvjob = arvados_cwl.ArvadosContainer(runner,
 619                                               runtimeContext,
 620                                               mock.MagicMock(),
 621                                               {},
 622                                               None,
 623                                               [],
 624                                               [],
 625                                               "testjob")
 626         arvjob.output_callback = mock.MagicMock()
 627         arvjob.collect_outputs = mock.MagicMock()
 628         arvjob.successCodes = [0]
 629         arvjob.outdir = "/var/spool/cwl"
 630         arvjob.output_ttl = 3600
 631         arvjob.collect_outputs.return_value = {"out": "stuff"}
 632
 633         arvjob.done({
 634             "state": "Final",
 635             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 636             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 637             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 638             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 639             "modified_at": "2017-05-26T12:01:22Z"
 640         })
 641
 642         rts_mock.assert_called_with(
 643             'error',
 644             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 645             '  ** log is empty **'
 646         )
 647         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 648
 649     # The test passes no builder.resources
 650     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 651     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 652     def test_mounts(self, keepdocker):
 653         runner = mock.MagicMock()
 654         runner.ignore_docker_for_reuse = False
 655         runner.intermediate_output_ttl = 0
 656         runner.secret_store = cwltool.secrets.SecretStore()
 657         runner.api._rootDesc = {"revision": "20210628"}
 658
 659         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 660         runner.api.collections().get().execute.return_value = {
 661             "portable_data_hash": "99999999999999999999999999999994+99",
 662             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 663
 664         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 665
 666         tool = cmap({
 667             "inputs": [
 668                 {"id": "p1",
 669                  "type": "Directory"}
 670             ],
 671             "outputs": [],
 672             "baseCommand": "ls",
 673             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 674             "id": "",
 675             "class": "CommandLineTool",
 676             "cwlVersion": "v1.2"
 677         })
 678
 679         loadingContext, runtimeContext = self.helper(runner)
 680         runtimeContext.name = "test_run_mounts"
 681
 682         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 683         arvtool.formatgraph = None
 684         job_order = {
 685             "p1": {
 686                 "class": "Directory",
 687                 "location": "keep:99999999999999999999999999999994+44",
 688                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 689                 "listing": [
 690                     {
 691                         "class": "File",
 692                         "location": "keep:99999999999999999999999999999994+44/file1",
 693                     },
 694                     {
 695                         "class": "File",
 696                         "location": "keep:99999999999999999999999999999994+44/file2",
 697                     }
 698                 ]
 699             }
 700         }
 701         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 702             j.run(runtimeContext)
 703             runner.api.container_requests().create.assert_called_with(
 704                 body=JsonDiffMatcher({
 705                     'environment': {
 706                         'HOME': '/var/spool/cwl',
 707                         'TMPDIR': '/tmp'
 708                     },
 709                     'name': 'test_run_mounts',
 710                     'runtime_constraints': {
 711                         'vcpus': 1,
 712                         'ram': 268435456
 713                     },
 714                     'use_existing': True,
 715                     'priority': 500,
 716                     'mounts': {
 717                         "/keep/99999999999999999999999999999994+44": {
 718                             "kind": "collection",
 719                             "portable_data_hash": "99999999999999999999999999999994+44",
 720                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 721                         },
 722                         '/tmp': {'kind': 'tmp',
 723                                  "capacity": 1073741824 },
 724                         '/var/spool/cwl': {'kind': 'tmp',
 725                                            "capacity": 1073741824 }
 726                     },
 727                     'state': 'Committed',
 728                     'output_name': 'Output for step test_run_mounts',
 729                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 730                     'output_path': '/var/spool/cwl',
 731                     'output_ttl': 0,
 732                     'container_image': '99999999999999999999999999999994+99',
 733                     'command': ['ls', '/var/spool/cwl'],
 734                     'cwd': '/var/spool/cwl',
 735                     'scheduling_parameters': {},
 736                     'properties': {},
 737                     'secret_mounts': {},
 738                     'output_storage_classes': ["default"]
 739                 }))
 740
 741     # The test passes no builder.resources
 742     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 743     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 744     def test_secrets(self, keepdocker):
 745         arvados_cwl.add_arv_hints()
 746         runner = mock.MagicMock()
 747         runner.ignore_docker_for_reuse = False
 748         runner.intermediate_output_ttl = 0
 749         runner.secret_store = cwltool.secrets.SecretStore()
 750         runner.api._rootDesc = {"revision": "20210628"}
 751
 752         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 753         runner.api.collections().get().execute.return_value = {
 754             "portable_data_hash": "99999999999999999999999999999993+99"}
 755
 756         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 757
 758         tool = cmap({"arguments": ["md5sum", "example.conf"],
 759                      "class": "CommandLineTool",
 760                      "cwlVersion": "v1.2",
 761                      "hints": [
 762                          {
 763                              "class": "http://commonwl.org/cwltool#Secrets",
 764                              "secrets": [
 765                                  "#secret_job.cwl/pw"
 766                              ]
 767                          }
 768                      ],
 769                      "id": "",
 770                      "inputs": [
 771                          {
 772                              "id": "#secret_job.cwl/pw",
 773                              "type": "string"
 774                          }
 775                      ],
 776                      "outputs": [
 777                      ],
 778                      "requirements": [
 779                          {
 780                              "class": "InitialWorkDirRequirement",
 781                              "listing": [
 782                                  {
 783                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 784                                      "entryname": "example.conf"
 785                                  }
 786                              ]
 787                          }
 788                      ]})
 789
 790         loadingContext, runtimeContext = self.helper(runner)
 791         runtimeContext.name = "test_secrets"
 792
 793         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 794         arvtool.formatgraph = None
 795
 796         job_order = {"pw": "blorp"}
 797         runner.secret_store.store(["pw"], job_order)
 798
 799         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 800             j.run(runtimeContext)
 801             runner.api.container_requests().create.assert_called_with(
 802                 body=JsonDiffMatcher({
 803                     'environment': {
 804                         'HOME': '/var/spool/cwl',
 805                         'TMPDIR': '/tmp'
 806                     },
 807                     'name': 'test_secrets',
 808                     'runtime_constraints': {
 809                         'vcpus': 1,
 810                         'ram': 268435456
 811                     },
 812                     'use_existing': True,
 813                     'priority': 500,
 814                     'mounts': {
 815                         '/tmp': {'kind': 'tmp',
 816                                  "capacity": 1073741824
 817                              },
 818                         '/var/spool/cwl': {'kind': 'tmp',
 819                                            "capacity": 1073741824 }
 820                     },
 821                     'state': 'Committed',
 822                     'output_name': 'Output for step test_secrets',
 823                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 824                     'output_path': '/var/spool/cwl',
 825                     'output_ttl': 0,
 826                     'container_image': '99999999999999999999999999999993+99',
 827                     'command': ['md5sum', 'example.conf'],
 828                     'cwd': '/var/spool/cwl',
 829                     'scheduling_parameters': {},
 830                     'properties': {},
 831                     "secret_mounts": {
 832                         "/var/spool/cwl/example.conf": {
 833                             "content": "username: user\npassword: blorp\n",
 834                             "kind": "text"
 835                         }
 836                     },
 837                     'output_storage_classes': ["default"]
 838                 }))
 839
 840     # The test passes no builder.resources
 841     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 842     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 843     def test_timelimit(self, keepdocker):
 844         runner = mock.MagicMock()
 845         runner.ignore_docker_for_reuse = False
 846         runner.intermediate_output_ttl = 0
 847         runner.secret_store = cwltool.secrets.SecretStore()
 848         runner.api._rootDesc = {"revision": "20210628"}
 849
 850         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 851         runner.api.collections().get().execute.return_value = {
 852             "portable_data_hash": "99999999999999999999999999999993+99"}
 853
 854         tool = cmap({
 855             "inputs": [],
 856             "outputs": [],
 857             "baseCommand": "ls",
 858             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 859             "id": "",
 860             "cwlVersion": "v1.2",
 861             "class": "CommandLineTool",
 862             "hints": [
 863                 {
 864                     "class": "ToolTimeLimit",
 865                     "timelimit": 42
 866                 }
 867             ]
 868         })
 869
 870         loadingContext, runtimeContext = self.helper(runner)
 871         runtimeContext.name = "test_timelimit"
 872
 873         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 874         arvtool.formatgraph = None
 875
 876         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 877             j.run(runtimeContext)
 878
 879         _, kwargs = runner.api.container_requests().create.call_args
 880         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 881
 882
 883     # The test passes no builder.resources
 884     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 885     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 886     def test_setting_storage_class(self, keepdocker):
 887         arv_docker_clear_cache()
 888
 889         runner = mock.MagicMock()
 890         runner.ignore_docker_for_reuse = False
 891         runner.intermediate_output_ttl = 0
 892         runner.secret_store = cwltool.secrets.SecretStore()
 893         runner.api._rootDesc = {"revision": "20210628"}
 894
 895         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 896         runner.api.collections().get().execute.return_value = {
 897             "portable_data_hash": "99999999999999999999999999999993+99"}
 898
 899         tool = cmap({
 900             "inputs": [],
 901             "outputs": [],
 902             "baseCommand": "ls",
 903             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 904             "id": "",
 905             "cwlVersion": "v1.2",
 906             "class": "CommandLineTool",
 907             "hints": [
 908                 {
 909                     "class": "http://arvados.org/cwl#OutputStorageClass",
 910                     "finalStorageClass": ["baz_sc", "qux_sc"],
 911                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 912                 }
 913             ]
 914         })
 915
 916         loadingContext, runtimeContext = self.helper(runner, True)
 917
 918         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 919         arvtool.formatgraph = None
 920
 921         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 922             j.run(runtimeContext)
 923             runner.api.container_requests().create.assert_called_with(
 924                 body=JsonDiffMatcher({
 925                     'environment': {
 926                         'HOME': '/var/spool/cwl',
 927                         'TMPDIR': '/tmp'
 928                     },
 929                     'name': 'test_run_True',
 930                     'runtime_constraints': {
 931                         'vcpus': 1,
 932                         'ram': 268435456
 933                     },
 934                     'use_existing': True,
 935                     'priority': 500,
 936                     'mounts': {
 937                         '/tmp': {'kind': 'tmp',
 938                                  "capacity": 1073741824
 939                              },
 940                         '/var/spool/cwl': {'kind': 'tmp',
 941                                            "capacity": 1073741824 }
 942                     },
 943                     'state': 'Committed',
 944                     'output_name': 'Output for step test_run_True',
 945                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 946                     'output_path': '/var/spool/cwl',
 947                     'output_ttl': 0,
 948                     'container_image': '99999999999999999999999999999993+99',
 949                     'command': ['ls', '/var/spool/cwl'],
 950                     'cwd': '/var/spool/cwl',
 951                     'scheduling_parameters': {},
 952                     'properties': {},
 953                     'secret_mounts': {},
 954                     'output_storage_classes': ["foo_sc", "bar_sc"]
 955                 }))
 956
 957
 958     # The test passes no builder.resources
 959     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 960     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 961     def test_setting_process_properties(self, keepdocker):
 962         arv_docker_clear_cache()
 963
 964         runner = mock.MagicMock()
 965         runner.ignore_docker_for_reuse = False
 966         runner.intermediate_output_ttl = 0
 967         runner.secret_store = cwltool.secrets.SecretStore()
 968         runner.api._rootDesc = {"revision": "20210628"}
 969
 970         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 971         runner.api.collections().get().execute.return_value = {
 972             "portable_data_hash": "99999999999999999999999999999993+99"}
 973
 974         tool = cmap({
 975             "inputs": [
 976                 {"id": "x", "type": "string"}],
 977             "outputs": [],
 978             "baseCommand": "ls",
 979             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 980             "id": "",
 981             "class": "CommandLineTool",
 982             "cwlVersion": "v1.2",
 983             "hints": [
 984             {
 985                 "class": "http://arvados.org/cwl#ProcessProperties",
 986                 "processProperties": [
 987                     {"propertyName": "foo",
 988                      "propertyValue": "bar"},
 989                     {"propertyName": "baz",
 990                      "propertyValue": "$(inputs.x)"},
 991                     {"propertyName": "quux",
 992                      "propertyValue": {
 993                          "q1": 1,
 994                          "q2": 2
 995                      }
 996                     }
 997                 ],
 998             }
 999         ]
1000         })
1001
1002         loadingContext, runtimeContext = self.helper(runner, True)
1003
1004         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1005         arvtool.formatgraph = None
1006
1007         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1008             j.run(runtimeContext)
1009             runner.api.container_requests().create.assert_called_with(
1010                 body=JsonDiffMatcher({
1011                     'environment': {
1012                         'HOME': '/var/spool/cwl',
1013                         'TMPDIR': '/tmp'
1014                     },
1015                     'name': 'test_run_True',
1016                     'runtime_constraints': {
1017                         'vcpus': 1,
1018                         'ram': 268435456
1019                     },
1020                     'use_existing': True,
1021                     'priority': 500,
1022                     'mounts': {
1023                         '/tmp': {'kind': 'tmp',
1024                                  "capacity": 1073741824
1025                              },
1026                         '/var/spool/cwl': {'kind': 'tmp',
1027                                            "capacity": 1073741824 }
1028                     },
1029                     'state': 'Committed',
1030                     'output_name': 'Output for step test_run_True',
1031                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1032                     'output_path': '/var/spool/cwl',
1033                     'output_ttl': 0,
1034                     'container_image': '99999999999999999999999999999993+99',
1035                     'command': ['ls', '/var/spool/cwl'],
1036                     'cwd': '/var/spool/cwl',
1037                     'scheduling_parameters': {},
1038                     'properties': {
1039                         "baz": "blorp",
1040                         "foo": "bar",
1041                         "quux": {
1042                             "q1": 1,
1043                             "q2": 2
1044                         }
1045                     },
1046                     'secret_mounts': {},
1047                     'output_storage_classes': ["default"]
1048                 }))
1049
1050
1051     # The test passes no builder.resources
1052     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1053     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1054     def test_cuda_requirement(self, keepdocker):
1055         arvados_cwl.add_arv_hints()
1056         arv_docker_clear_cache()
1057
1058         runner = mock.MagicMock()
1059         runner.ignore_docker_for_reuse = False
1060         runner.intermediate_output_ttl = 0
1061         runner.secret_store = cwltool.secrets.SecretStore()
1062         runner.api._rootDesc = {"revision": "20210628"}
1063
1064         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1065         runner.api.collections().get().execute.return_value = {
1066             "portable_data_hash": "99999999999999999999999999999993+99"}
1067
1068         test_cwl_req = [{
1069                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1070                 "cudaVersionMin": "11.0",
1071                 "cudaComputeCapability": "9.0",
1072             }, {
1073                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1074                 "cudaVersionMin": "11.0",
1075                 "cudaComputeCapability": "9.0",
1076                 "cudaDeviceCountMin": 2
1077             }, {
1078                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1079                 "cudaVersionMin": "11.0",
1080                 "cudaComputeCapability": ["4.0", "5.0"],
1081                 "cudaDeviceCountMin": 2
1082             }]
1083
1084         test_arv_req = [{
1085             'device_count': 1,
1086             'driver_version': "11.0",
1087             'hardware_capability': "9.0"
1088         }, {
1089             'device_count': 2,
1090             'driver_version': "11.0",
1091             'hardware_capability': "9.0"
1092         }, {
1093             'device_count': 2,
1094             'driver_version': "11.0",
1095             'hardware_capability': "4.0"
1096         }]
1097
1098         for test_case in range(0, len(test_cwl_req)):
1099
1100             tool = cmap({
1101                 "inputs": [],
1102                 "outputs": [],
1103                 "baseCommand": "nvidia-smi",
1104                 "arguments": [],
1105                 "id": "",
1106                 "cwlVersion": "v1.2",
1107                 "class": "CommandLineTool",
1108                 "requirements": [test_cwl_req[test_case]]
1109             })
1110
1111             loadingContext, runtimeContext = self.helper(runner, True)
1112
1113             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1114             arvtool.formatgraph = None
1115
1116             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1117                 j.run(runtimeContext)
1118                 runner.api.container_requests().create.assert_called_with(
1119                     body=JsonDiffMatcher({
1120                         'environment': {
1121                             'HOME': '/var/spool/cwl',
1122                             'TMPDIR': '/tmp'
1123                         },
1124                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1125                         'runtime_constraints': {
1126                             'vcpus': 1,
1127                             'ram': 268435456,
1128                             'cuda': test_arv_req[test_case]
1129                         },
1130                         'use_existing': True,
1131                         'priority': 500,
1132                         'mounts': {
1133                             '/tmp': {'kind': 'tmp',
1134                                      "capacity": 1073741824
1135                                  },
1136                             '/var/spool/cwl': {'kind': 'tmp',
1137                                                "capacity": 1073741824 }
1138                         },
1139                         'state': 'Committed',
1140                         'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1141                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1142                         'output_path': '/var/spool/cwl',
1143                         'output_ttl': 0,
1144                         'container_image': '99999999999999999999999999999993+99',
1145                         'command': ['nvidia-smi'],
1146                         'cwd': '/var/spool/cwl',
1147                         'scheduling_parameters': {},
1148                         'properties': {},
1149                         'secret_mounts': {},
1150                         'output_storage_classes': ["default"]
1151                     }))
1152
1153
1154     # The test passes no builder.resources
1155     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1156     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1157     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1158     def test_match_local_docker(self, keepdocker, determine_image_id):
1159         arvados_cwl.add_arv_hints()
1160         arv_docker_clear_cache()
1161
1162         runner = mock.MagicMock()
1163         runner.ignore_docker_for_reuse = False
1164         runner.intermediate_output_ttl = 0
1165         runner.secret_store = cwltool.secrets.SecretStore()
1166         runner.api._rootDesc = {"revision": "20210628"}
1167
1168         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1169                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1170         determine_image_id.side_effect = lambda x: "123"
1171         def execute(uuid):
1172             ex = mock.MagicMock()
1173             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1174                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1175             ex.execute.return_value = lookup[uuid]
1176             return ex
1177         runner.api.collections().get.side_effect = execute
1178
1179         tool = cmap({
1180             "inputs": [],
1181             "outputs": [],
1182             "baseCommand": "echo",
1183             "arguments": [],
1184             "id": "",
1185             "cwlVersion": "v1.0",
1186             "class": "org.w3id.cwl.cwl.CommandLineTool"
1187         })
1188
1189         loadingContext, runtimeContext = self.helper(runner, True)
1190
1191         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1192         arvtool.formatgraph = None
1193
1194         container_request = {
1195             'environment': {
1196                 'HOME': '/var/spool/cwl',
1197                 'TMPDIR': '/tmp'
1198             },
1199             'name': 'test_run_True',
1200             'runtime_constraints': {
1201                 'vcpus': 1,
1202                 'ram': 1073741824,
1203             },
1204             'use_existing': True,
1205             'priority': 500,
1206             'mounts': {
1207                 '/tmp': {'kind': 'tmp',
1208                          "capacity": 1073741824
1209                          },
1210                 '/var/spool/cwl': {'kind': 'tmp',
1211                                    "capacity": 1073741824 }
1212             },
1213             'state': 'Committed',
1214             'output_name': 'Output for step test_run_True',
1215             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1216             'output_path': '/var/spool/cwl',
1217             'output_ttl': 0,
1218             'container_image': '99999999999999999999999999999994+99',
1219             'command': ['echo'],
1220             'cwd': '/var/spool/cwl',
1221             'scheduling_parameters': {},
1222             'properties': {},
1223             'secret_mounts': {},
1224             'output_storage_classes': ["default"]
1225         }
1226
1227         runtimeContext.match_local_docker = False
1228         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1229             j.run(runtimeContext)
1230             runner.api.container_requests().create.assert_called_with(
1231                 body=JsonDiffMatcher(container_request))
1232
1233         arv_docker_clear_cache()
1234         runtimeContext.match_local_docker = True
1235         container_request['container_image'] = '99999999999999999999999999999993+99'
1236         container_request['name'] = 'test_run_True_2'
1237         container_request['output_name'] = 'Output for step test_run_True_2'
1238         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1239             j.run(runtimeContext)
1240             runner.api.container_requests().create.assert_called_with(
1241                 body=JsonDiffMatcher(container_request))
1242
1243
1244     # The test passes no builder.resources
1245     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1246     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1247     def test_run_preemptible_hint(self, keepdocker):
1248         arvados_cwl.add_arv_hints()
1249         for enable_preemptible in (None, True, False):
1250             for preemptible_hint in (None, True, False):
1251                 arv_docker_clear_cache()
1252
1253                 runner = mock.MagicMock()
1254                 runner.ignore_docker_for_reuse = False
1255                 runner.intermediate_output_ttl = 0
1256                 runner.secret_store = cwltool.secrets.SecretStore()
1257                 runner.api._rootDesc = {"revision": "20210628"}
1258
1259                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1260                 runner.api.collections().get().execute.return_value = {
1261                     "portable_data_hash": "99999999999999999999999999999993+99"}
1262
1263                 if preemptible_hint is not None:
1264                     hints = [{
1265                         "class": "http://arvados.org/cwl#UsePreemptible",
1266                         "usePreemptible": preemptible_hint
1267                     }]
1268                 else:
1269                     hints = []
1270
1271                 tool = cmap({
1272                     "inputs": [],
1273                     "outputs": [],
1274                     "baseCommand": "ls",
1275                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1276                     "id": "",
1277                     "class": "CommandLineTool",
1278                     "cwlVersion": "v1.2",
1279                     "hints": hints
1280                 })
1281
1282                 loadingContext, runtimeContext = self.helper(runner)
1283
1284                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1285                 runtimeContext.enable_preemptible = enable_preemptible
1286
1287                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1288                 arvtool.formatgraph = None
1289
1290                 # Test the interactions between --enable/disable-preemptible
1291                 # and UsePreemptible hint
1292
1293                 if enable_preemptible is None:
1294                     if preemptible_hint is None:
1295                         sched = {}
1296                     else:
1297                         sched = {'preemptible': preemptible_hint}
1298                 else:
1299                     if preemptible_hint is None:
1300                         sched = {'preemptible': enable_preemptible}
1301                     else:
1302                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1303
1304                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1305                     j.run(runtimeContext)
1306                     runner.api.container_requests().create.assert_called_with(
1307                         body=JsonDiffMatcher({
1308                             'environment': {
1309                                 'HOME': '/var/spool/cwl',
1310                                 'TMPDIR': '/tmp'
1311                             },
1312                             'name': runtimeContext.name,
1313                             'runtime_constraints': {
1314                                 'vcpus': 1,
1315                                 'ram': 268435456
1316                             },
1317                             'use_existing': True,
1318                             'priority': 500,
1319                             'mounts': {
1320                                 '/tmp': {'kind': 'tmp',
1321                                          "capacity": 1073741824
1322                                      },
1323                                 '/var/spool/cwl': {'kind': 'tmp',
1324                                                    "capacity": 1073741824 }
1325                             },
1326                             'state': 'Committed',
1327                             'output_name': 'Output for step '+runtimeContext.name,
1328                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1329                             'output_path': '/var/spool/cwl',
1330                             'output_ttl': 0,
1331                             'container_image': '99999999999999999999999999999993+99',
1332                             'command': ['ls', '/var/spool/cwl'],
1333                             'cwd': '/var/spool/cwl',
1334                             'scheduling_parameters': sched,
1335                             'properties': {},
1336                             'secret_mounts': {},
1337                             'output_storage_classes': ["default"]
1338                         }))
1339
1340
1341
1342 class TestWorkflow(unittest.TestCase):
1343     def setUp(self):
1344         cwltool.process._names = set()
1345         arv_docker_clear_cache()
1346
1347     def helper(self, runner, enable_reuse=True):
1348         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1349
1350         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1351                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1352
1353         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1354         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1355         document_loader.fetch_text = document_loader.fetcher.fetch_text
1356         document_loader.check_exists = document_loader.fetcher.check_exists
1357
1358         loadingContext = arvados_cwl.context.ArvLoadingContext(
1359             {"avsc_names": avsc_names,
1360              "basedir": "",
1361              "make_fs_access": make_fs_access,
1362              "loader": document_loader,
1363              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1364              "construct_tool_object": runner.arv_make_tool})
1365         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1366             {"work_api": "containers",
1367              "basedir": "",
1368              "name": "test_run_wf_"+str(enable_reuse),
1369              "make_fs_access": make_fs_access,
1370              "tmpdir": "/tmp",
1371              "enable_reuse": enable_reuse,
1372              "priority": 500})
1373
1374         return loadingContext, runtimeContext
1375
1376     # The test passes no builder.resources
1377     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1378     @mock.patch("arvados.collection.CollectionReader")
1379     @mock.patch("arvados.collection.Collection")
1380     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1381     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1382         arvados_cwl.add_arv_hints()
1383
1384         api = mock.MagicMock()
1385         api._rootDesc = get_rootDesc()
1386
1387         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1388         self.assertEqual(runner.work_api, 'containers')
1389
1390         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1391         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1392         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1393                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1394
1395         runner.api.containers().current().execute.return_value = {}
1396
1397         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1398         runner.ignore_docker_for_reuse = False
1399         runner.num_retries = 0
1400         runner.secret_store = cwltool.secrets.SecretStore()
1401
1402         loadingContext, runtimeContext = self.helper(runner)
1403         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1404
1405         mockcollectionreader().exists.return_value = True
1406
1407         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1408         metadata["cwlVersion"] = tool["cwlVersion"]
1409
1410         mockc = mock.MagicMock()
1411         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1412         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1413
1414         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1415         arvtool.formatgraph = None
1416         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1417
1418         next(it).run(runtimeContext)
1419         next(it).run(runtimeContext)
1420
1421         with open("tests/wf/scatter2_subwf.cwl") as f:
1422             subwf = StripYAMLComments(f.read()).rstrip()
1423
1424         runner.api.container_requests().create.assert_called_with(
1425             body=JsonDiffMatcher({
1426                 "command": [
1427                     "cwltool",
1428                     "--no-container",
1429                     "--move-outputs",
1430                     "--preserve-entire-environment",
1431                     "workflow.cwl",
1432                     "cwl.input.yml"
1433                 ],
1434                 "container_image": "99999999999999999999999999999993+99",
1435                 "cwd": "/var/spool/cwl",
1436                 "environment": {
1437                     "HOME": "/var/spool/cwl",
1438                     "TMPDIR": "/tmp"
1439                 },
1440                 "mounts": {
1441                     "/keep/99999999999999999999999999999999+118": {
1442                         "kind": "collection",
1443                         "portable_data_hash": "99999999999999999999999999999999+118"
1444                     },
1445                     "/tmp": {
1446                         "capacity": 1073741824,
1447                         "kind": "tmp"
1448                     },
1449                     "/var/spool/cwl": {
1450                         "capacity": 1073741824,
1451                         "kind": "tmp"
1452                     },
1453                     "/var/spool/cwl/cwl.input.yml": {
1454                         "kind": "collection",
1455                         "path": "cwl.input.yml",
1456                         "portable_data_hash": "99999999999999999999999999999996+99"
1457                     },
1458                     "/var/spool/cwl/workflow.cwl": {
1459                         "kind": "collection",
1460                         "path": "workflow.cwl",
1461                         "portable_data_hash": "99999999999999999999999999999996+99"
1462                     },
1463                     "stdout": {
1464                         "kind": "file",
1465                         "path": "/var/spool/cwl/cwl.output.json"
1466                     }
1467                 },
1468                 "name": "scatterstep",
1469                 "output_name": "Output for step scatterstep",
1470                 "output_path": "/var/spool/cwl",
1471                 "output_ttl": 0,
1472                 "priority": 500,
1473                 "properties": {},
1474                 "runtime_constraints": {
1475                     "ram": 1073741824,
1476                     "vcpus": 1
1477                 },
1478                 "scheduling_parameters": {},
1479                 "secret_mounts": {},
1480                 "state": "Committed",
1481                 "use_existing": True,
1482                 'output_storage_classes': ["default"]
1483             }))
1484         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1485         mockc.open().__enter__().write.assert_has_calls([mock.call(
1486 '''{
1487   "fileblub": {
1488     "basename": "token.txt",
1489     "class": "File",
1490     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1491     "size": 0
1492   },
1493   "sleeptime": 5
1494 }''')])
1495
1496     # The test passes no builder.resources
1497     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1498     @mock.patch("arvados.collection.CollectionReader")
1499     @mock.patch("arvados.collection.Collection")
1500     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1501     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1502         arvados_cwl.add_arv_hints()
1503
1504         api = mock.MagicMock()
1505         api._rootDesc = get_rootDesc()
1506
1507         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1508         self.assertEqual(runner.work_api, 'containers')
1509
1510         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1511         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1512                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1513         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1514                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1515
1516         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1517         runner.ignore_docker_for_reuse = False
1518         runner.num_retries = 0
1519         runner.secret_store = cwltool.secrets.SecretStore()
1520
1521         loadingContext, runtimeContext = self.helper(runner)
1522         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1523         loadingContext.do_update = True
1524         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1525
1526         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1527
1528         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1529         arvtool.formatgraph = None
1530         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1531
1532         next(it).run(runtimeContext)
1533         next(it).run(runtimeContext)
1534
1535         with open("tests/wf/echo-subwf.cwl") as f:
1536             subwf = StripYAMLComments(f.read())
1537
1538         runner.api.container_requests().create.assert_called_with(
1539             body=JsonDiffMatcher({
1540                 'output_ttl': 0,
1541                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1542                 'scheduling_parameters': {},
1543                 'name': u'echo-subwf',
1544                 'secret_mounts': {},
1545                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1546                 'properties': {},
1547                 'priority': 500,
1548                 'mounts': {
1549                     '/var/spool/cwl/cwl.input.yml': {
1550                         'portable_data_hash': '99999999999999999999999999999996+99',
1551                         'kind': 'collection',
1552                         'path': 'cwl.input.yml'
1553                     },
1554                     '/var/spool/cwl/workflow.cwl': {
1555                         'portable_data_hash': '99999999999999999999999999999996+99',
1556                         'kind': 'collection',
1557                         'path': 'workflow.cwl'
1558                     },
1559                     'stdout': {
1560                         'path': '/var/spool/cwl/cwl.output.json',
1561                         'kind': 'file'
1562                     },
1563                     '/tmp': {
1564                         'kind': 'tmp',
1565                         'capacity': 1073741824
1566                     }, '/var/spool/cwl': {
1567                         'kind': 'tmp',
1568                         'capacity': 3221225472
1569                     }
1570                 },
1571                 'state': 'Committed',
1572                 'output_path': '/var/spool/cwl',
1573                 'container_image': '99999999999999999999999999999993+99',
1574                 'command': [
1575                     u'cwltool',
1576                     u'--no-container',
1577                     u'--move-outputs',
1578                     u'--preserve-entire-environment',
1579                     u'workflow.cwl',
1580                     u'cwl.input.yml'
1581                 ],
1582                 'use_existing': True,
1583                 'output_name': u'Output for step echo-subwf',
1584                 'cwd': '/var/spool/cwl',
1585                 'output_storage_classes': ["default"]
1586             }))
1587
1588     def test_default_work_api(self):
1589         arvados_cwl.add_arv_hints()
1590
1591         api = mock.MagicMock()
1592         api._rootDesc = copy.deepcopy(get_rootDesc())
1593         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1594         self.assertEqual(runner.work_api, 'containers')