sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 import arvados_cwl.util
   8 import copy
   9 import arvados.config
  10 import logging
  11 import unittest
  12 import os
  13 import functools
  14 import threading
  15 import cwltool.process
  16 import cwltool.secrets
  17 import cwltool.load_tool
  18 from cwltool.update import INTERNAL_VERSION
  19 from schema_salad.ref_resolver import Loader
  20 from schema_salad.sourceline import cmap
  21 import io
  22 from parameterized import parameterized
  23
  24 from unittest import mock
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63
  64     def tearDown(self):
  65         root_logger = logging.getLogger('')
  66
  67         # Remove existing RuntimeStatusLoggingHandlers if they exist
  68         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  69         root_logger.handlers = handlers
  70
  71     def helper(self, runner, enable_reuse=True):
  72         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  73
  74         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  75                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  76         fs_access = mock.MagicMock()
  77         fs_access.exists.return_value = True
  78
  79         loadingContext = arvados_cwl.context.ArvLoadingContext(
  80             {"avsc_names": avsc_names,
  81              "basedir": "",
  82              "make_fs_access": make_fs_access,
  83              "construct_tool_object": runner.arv_make_tool,
  84              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  85              "loader": Loader({}),
  86              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  87              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__,
  88              })
  89         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  90             {"work_api": "containers",
  91              "basedir": "",
  92              "name": "test_run_"+str(enable_reuse),
  93              "make_fs_access": make_fs_access,
  94              "tmpdir": "/tmp",
  95              "outdir": "/tmp",
  96              "enable_reuse": enable_reuse,
  97              "priority": 500,
  98              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
  99              "workflow_eval_lock": threading.Condition(threading.RLock())
 100             })
 101
 102         if isinstance(runner, mock.MagicMock):
 103             def make_tool(toolpath_object, loadingContext):
 104                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 105             runner.arv_make_tool.side_effect = make_tool
 106
 107         return loadingContext, runtimeContext
 108
 109     # Helper function to set up the ArvCwlExecutor to use the containers api
 110     # and test that the RuntimeStatusLoggingHandler is set up correctly
 111     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 112         api = mock.MagicMock()
 113         api._rootDesc = copy.deepcopy(get_rootDesc())
 114
 115         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 116         # adds the logging handler that will call runtime_status_update() mock
 117         self.assertFalse(gcc_mock.called)
 118         runner = arvados_cwl.ArvCwlExecutor(api)
 119         self.assertEqual(runner.work_api, 'containers')
 120         root_logger = logging.getLogger('')
 121         handlerClasses = [h.__class__ for h in root_logger.handlers]
 122         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 123         return runner
 124
 125     # The test passes no builder.resources
 126     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 127     @parameterized.expand([
 128         (True,),
 129         (False,),
 130     ])
 131     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 132     def test_run(self, enable_reuse, keepdocker):
 133         runner = mock.MagicMock()
 134         runner.ignore_docker_for_reuse = False
 135         runner.intermediate_output_ttl = 0
 136         runner.secret_store = cwltool.secrets.SecretStore()
 137         runner.api._rootDesc = {"revision": "20210628"}
 138         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 139
 140         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 141         runner.api.collections().get().execute.return_value = {
 142             "portable_data_hash": "99999999999999999999999999999993+99"}
 143
 144         tool = cmap({
 145             "inputs": [],
 146             "outputs": [],
 147             "baseCommand": "ls",
 148             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 149             "id": "",
 150             "class": "CommandLineTool",
 151             "cwlVersion": "v1.2"
 152         })
 153
 154         loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 155
 156         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 157         arvtool.formatgraph = None
 158
 159         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 160             j.run(runtimeContext)
 161             runner.api.container_requests().create.assert_called_with(
 162                 body=JsonDiffMatcher({
 163                     'environment': {
 164                         'HOME': '/var/spool/cwl',
 165                         'TMPDIR': '/tmp'
 166                     },
 167                     'name': 'test_run_'+str(enable_reuse),
 168                     'runtime_constraints': {
 169                         'vcpus': 1,
 170                         'ram': 268435456
 171                     },
 172                     'use_existing': enable_reuse,
 173                     'priority': 500,
 174                     'mounts': {
 175                         '/tmp': {'kind': 'tmp',
 176                                  "capacity": 1073741824
 177                              },
 178                         '/var/spool/cwl': {'kind': 'tmp',
 179                                            "capacity": 1073741824 }
 180                     },
 181                     'state': 'Committed',
 182                     'output_name': 'Output from step test_run_'+str(enable_reuse),
 183                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                     'output_path': '/var/spool/cwl',
 185                     'output_ttl': 0,
 186                     'container_image': '99999999999999999999999999999993+99',
 187                     'command': ['ls', '/var/spool/cwl'],
 188                     'cwd': '/var/spool/cwl',
 189                     'scheduling_parameters': {},
 190                     'properties': {'cwl_input': {}},
 191                     'secret_mounts': {},
 192                     'output_storage_classes': ["default"]
 193                 }))
 194
 195     # The test passes some fields in builder.resources
 196     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 198     def test_resource_requirements(self, keepdocker):
 199         arvados_cwl.add_arv_hints()
 200         runner = mock.MagicMock()
 201         runner.ignore_docker_for_reuse = False
 202         runner.intermediate_output_ttl = 3600
 203         runner.secret_store = cwltool.secrets.SecretStore()
 204         runner.api._rootDesc = {"revision": "20210628"}
 205         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 206
 207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 208         runner.api.collections().get().execute.return_value = {
 209             "portable_data_hash": "99999999999999999999999999999993+99"}
 210
 211         tool = cmap({
 212             "inputs": [],
 213             "outputs": [],
 214             "hints": [{
 215                 "class": "ResourceRequirement",
 216                 "coresMin": 3,
 217                 "ramMin": 3000,
 218                 "tmpdirMin": 4000,
 219                 "outdirMin": 5000
 220             }, {
 221                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 222                 "keep_cache": 512
 223             }, {
 224                 "class": "http://arvados.org/cwl#APIRequirement",
 225             }, {
 226                 "class": "http://arvados.org/cwl#PartitionRequirement",
 227                 "partition": "blurb"
 228             }, {
 229                 "class": "http://arvados.org/cwl#IntermediateOutput",
 230                 "outputTTL": 7200
 231             }, {
 232                 "class": "WorkReuse",
 233                 "enableReuse": False
 234             }],
 235             "baseCommand": "ls",
 236             "id": "",
 237             "class": "CommandLineTool",
 238             "cwlVersion": "v1.2"
 239         })
 240
 241         loadingContext, runtimeContext = self.helper(runner)
 242         runtimeContext.name = "test_resource_requirements"
 243
 244         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 245         arvtool.formatgraph = None
 246         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 247             j.run(runtimeContext)
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         call_body_expected = {
 252             'environment': {
 253                 'HOME': '/var/spool/cwl',
 254                 'TMPDIR': '/tmp'
 255             },
 256             'name': 'test_resource_requirements',
 257             'runtime_constraints': {
 258                 'vcpus': 3,
 259                 'ram': 3145728000,
 260                 'keep_cache_ram': 536870912,
 261                 'API': True
 262             },
 263             'use_existing': False,
 264             'priority': 500,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 4194304000 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 5242880000 }
 270             },
 271             'state': 'Committed',
 272             'output_name': 'Output from step test_resource_requirements',
 273             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 274             'output_path': '/var/spool/cwl',
 275             'output_ttl': 7200,
 276             'container_image': '99999999999999999999999999999993+99',
 277             'command': ['ls'],
 278             'cwd': '/var/spool/cwl',
 279             'scheduling_parameters': {
 280                 'partitions': ['blurb']
 281             },
 282             'properties': {'cwl_input': {}},
 283             'secret_mounts': {},
 284             'output_storage_classes': ["default"]
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # The test passes some fields in builder.resources
 294     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 296     @mock.patch("arvados.collection.Collection")
 297     def test_initial_work_dir(self, collection_mock, keepdocker):
 298         runner = mock.MagicMock()
 299         runner.ignore_docker_for_reuse = False
 300         runner.intermediate_output_ttl = 0
 301         runner.secret_store = cwltool.secrets.SecretStore()
 302         runner.api._rootDesc = {"revision": "20210628"}
 303         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 304
 305         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 306         runner.api.collections().get().execute.return_value = {
 307             "portable_data_hash": "99999999999999999999999999999993+99"}
 308
 309         sourcemock = mock.MagicMock()
 310         def get_collection_mock(p):
 311             if "/" in p:
 312                 return (sourcemock, p.split("/", 1)[1])
 313             else:
 314                 return (sourcemock, "")
 315         runner.fs_access.get_collection.side_effect = get_collection_mock
 316
 317         vwdmock = mock.MagicMock()
 318         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 319
 320         tool = cmap({
 321             "inputs": [],
 322             "outputs": [],
 323             "hints": [{
 324                 "class": "InitialWorkDirRequirement",
 325                 "listing": [{
 326                     "class": "File",
 327                     "basename": "foo",
 328                     "location": "keep:99999999999999999999999999999995+99/bar"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "foo2",
 333                     "location": "keep:99999999999999999999999999999995+99"
 334                 },
 335                 {
 336                     "class": "File",
 337                     "basename": "filename",
 338                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 339                 },
 340                 {
 341                     "class": "Directory",
 342                     "basename": "subdir",
 343                     "location": "keep:99999999999999999999999999999995+99/subdir"
 344                 }                        ]
 345             }],
 346             "baseCommand": "ls",
 347             "class": "CommandLineTool",
 348             "cwlVersion": "v1.2",
 349             "id": ""
 350         })
 351
 352         loadingContext, runtimeContext = self.helper(runner)
 353         runtimeContext.name = "test_initial_work_dir"
 354
 355         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 356
 357         arvtool.formatgraph = None
 358         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 359             j.run(runtimeContext)
 360
 361         call_args, call_kwargs = runner.api.container_requests().create.call_args
 362
 363         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 367
 368         call_body_expected = {
 369             'environment': {
 370                 'HOME': '/var/spool/cwl',
 371                 'TMPDIR': '/tmp'
 372             },
 373             'name': 'test_initial_work_dir',
 374             'runtime_constraints': {
 375                 'vcpus': 1,
 376                 'ram': 268435456
 377             },
 378             'use_existing': True,
 379             'priority': 500,
 380             'mounts': {
 381                 '/tmp': {'kind': 'tmp',
 382                          "capacity": 1073741824 },
 383                 '/var/spool/cwl': {'kind': 'tmp',
 384                                    "capacity": 1073741824 },
 385                 '/var/spool/cwl/foo': {
 386                     'kind': 'collection',
 387                     'path': 'foo',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/foo2': {
 391                     'kind': 'collection',
 392                     'path': 'foo2',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 },
 395                 '/var/spool/cwl/filename': {
 396                     'kind': 'collection',
 397                     'path': 'filename',
 398                     'portable_data_hash': '99999999999999999999999999999996+99'
 399                 },
 400                 '/var/spool/cwl/subdir': {
 401                     'kind': 'collection',
 402                     'path': 'subdir',
 403                     'portable_data_hash': '99999999999999999999999999999996+99'
 404                 }
 405             },
 406             'state': 'Committed',
 407             'output_name': 'Output from step test_initial_work_dir',
 408             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409             'output_path': '/var/spool/cwl',
 410             'output_ttl': 0,
 411             'container_image': '99999999999999999999999999999993+99',
 412             'command': ['ls'],
 413             'cwd': '/var/spool/cwl',
 414             'scheduling_parameters': {
 415             },
 416             'properties': {'cwl_input': {}},
 417             'secret_mounts': {},
 418             'output_storage_classes': ["default"]
 419         }
 420
 421         call_body = call_kwargs.get('body', None)
 422         self.assertNotEqual(None, call_body)
 423         for key in call_body:
 424             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 425
 426
 427     # Test redirecting stdin/stdout/stderr
 428     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 429     def test_redirects(self, keepdocker):
 430         runner = mock.MagicMock()
 431         runner.ignore_docker_for_reuse = False
 432         runner.intermediate_output_ttl = 0
 433         runner.secret_store = cwltool.secrets.SecretStore()
 434         runner.api._rootDesc = {"revision": "20210628"}
 435         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 436
 437         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 438         runner.api.collections().get().execute.return_value = {
 439             "portable_data_hash": "99999999999999999999999999999993+99"}
 440
 441         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 442
 443         tool = cmap({
 444             "inputs": [],
 445             "outputs": [],
 446             "baseCommand": "ls",
 447             "stdout": "stdout.txt",
 448             "stderr": "stderr.txt",
 449             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 450             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 451             "id": "",
 452             "class": "CommandLineTool",
 453             "cwlVersion": "v1.2"
 454         })
 455
 456         loadingContext, runtimeContext = self.helper(runner)
 457         runtimeContext.name = "test_run_redirect"
 458
 459         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 460         arvtool.formatgraph = None
 461         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 462             j.run(runtimeContext)
 463             runner.api.container_requests().create.assert_called_with(
 464                 body=JsonDiffMatcher({
 465                     'environment': {
 466                         'HOME': '/var/spool/cwl',
 467                         'TMPDIR': '/tmp'
 468                     },
 469                     'name': 'test_run_redirect',
 470                     'runtime_constraints': {
 471                         'vcpus': 1,
 472                         'ram': 268435456
 473                     },
 474                     'use_existing': True,
 475                     'priority': 500,
 476                     'mounts': {
 477                         '/tmp': {'kind': 'tmp',
 478                                  "capacity": 1073741824 },
 479                         '/var/spool/cwl': {'kind': 'tmp',
 480                                            "capacity": 1073741824 },
 481                         "stderr": {
 482                             "kind": "file",
 483                             "path": "/var/spool/cwl/stderr.txt"
 484                         },
 485                         "stdin": {
 486                             "kind": "collection",
 487                             "path": "file.txt",
 488                             "portable_data_hash": "99999999999999999999999999999996+99"
 489                         },
 490                         "stdout": {
 491                             "kind": "file",
 492                             "path": "/var/spool/cwl/stdout.txt"
 493                         },
 494                     },
 495                     'state': 'Committed',
 496                     "output_name": "Output from step test_run_redirect",
 497                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 498                     'output_path': '/var/spool/cwl',
 499                     'output_ttl': 0,
 500                     'container_image': '99999999999999999999999999999993+99',
 501                     'command': ['ls', '/var/spool/cwl'],
 502                     'cwd': '/var/spool/cwl',
 503                     'scheduling_parameters': {},
 504                     'properties': {'cwl_input': {}},
 505                     'secret_mounts': {},
 506                     'output_storage_classes': ["default"]
 507                 }))
 508
 509     @mock.patch("arvados.collection.Collection")
 510     def test_done(self, col):
 511         api = mock.MagicMock()
 512
 513         runner = mock.MagicMock()
 514         runner.api = api
 515         runner.num_retries = 0
 516         runner.ignore_docker_for_reuse = False
 517         runner.intermediate_output_ttl = 0
 518         runner.secret_store = cwltool.secrets.SecretStore()
 519
 520         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 521
 522         runner.api.containers().get().execute.return_value = {"state":"Complete",
 523                                                               "output": "abc+123",
 524                                                               "exit_code": 0}
 525
 526         # Need to noop-out the close method otherwise it gets
 527         # discarded when closed and we can't call getvalue() to check
 528         # it.
 529         class NoopCloseStringIO(io.StringIO):
 530             def close(self):
 531                 pass
 532
 533         usage_report = NoopCloseStringIO()
 534         def colreader_action(name, mode):
 535             nonlocal usage_report
 536             if name == "node.json":
 537                 return io.StringIO("""{
 538     "ProviderType": "c5.large",
 539     "VCPUs": 2,
 540     "RAM": 4294967296,
 541     "IncludedScratch": 8000000000000,
 542     "AddedScratch": 0,
 543     "Price": 0.085,
 544     "Preemptible": false,
 545     "CUDA": {
 546         "DriverVersion": "",
 547         "HardwareCapability": "",
 548         "DeviceCount": 0
 549     }
 550 }""")
 551             if name == 'crunchstat.txt':
 552                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 553             if name == 'arv-mount.txt':
 554                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 555             if name == 'usage_report.html':
 556                 return usage_report
 557             return None
 558
 559         col().open.side_effect = colreader_action
 560         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 561
 562         loadingContext, runtimeContext = self.helper(runner)
 563
 564         arvjob = arvados_cwl.ArvadosContainer(runner,
 565                                               runtimeContext,
 566                                               [],
 567                                               mock.MagicMock(),
 568                                               {},
 569                                               None,
 570                                               [],
 571                                               [],
 572                                               "testjob")
 573         arvjob.output_callback = mock.MagicMock()
 574         arvjob.collect_outputs = mock.MagicMock()
 575         arvjob.successCodes = [0]
 576         arvjob.outdir = "/var/spool/cwl"
 577         arvjob.output_ttl = 3600
 578         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 579
 580         arvjob.collect_outputs.return_value = {"out": "stuff"}
 581
 582         arvjob.done({
 583             "state": "Final",
 584             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 585             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 586             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 587             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 588             "modified_at": "2017-05-26T12:01:22Z",
 589             "properties": {},
 590             "name": "testjob"
 591         })
 592
 593         self.assertFalse(api.collections().create.called)
 594         self.assertFalse(runner.runtime_status_error.called)
 595
 596         # Assert that something was written to the usage report
 597         self.assertTrue(len(usage_report.getvalue()) > 0)
 598
 599         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 600         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 601         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 602
 603         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 604                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 605
 606
 607     # Test to make sure we dont call runtime_status_update if we already did
 608     # some where higher up in the call stack
 609     @mock.patch("arvados_cwl.util.get_current_container")
 610     def test_recursive_runtime_status_update(self, gcc_mock):
 611         self.setup_and_test_container_executor_and_logging(gcc_mock)
 612         root_logger = logging.getLogger('')
 613
 614         # get_current_container is invoked when we call runtime_status_update
 615         # so try and log again!
 616         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 617         try:
 618             root_logger.error("First Error")
 619         except RuntimeError:
 620             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 621
 622
 623     # Test to make sure that an exception raised from
 624     # get_current_container doesn't cause the logger to raise an
 625     # exception
 626     @mock.patch("arvados_cwl.util.get_current_container")
 627     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 628         self.setup_and_test_container_executor_and_logging(gcc_mock)
 629         root_logger = logging.getLogger('')
 630
 631         # get_current_container is invoked when we call
 632         # runtime_status_update, it is going to also raise an
 633         # exception.
 634         gcc_mock.side_effect = Exception("Second Error")
 635         try:
 636             root_logger.error("First Error")
 637         except Exception:
 638             self.fail("Exception in logger should not propagate")
 639         self.assertTrue(gcc_mock.called)
 640
 641     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 642     @mock.patch("arvados_cwl.util.get_current_container")
 643     @mock.patch("arvados.collection.CollectionReader")
 644     @mock.patch("arvados.collection.Collection")
 645     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 646         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 647
 648         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 649         self.assertTrue(gcc_mock.called)
 650
 651         runner.num_retries = 0
 652         runner.ignore_docker_for_reuse = False
 653         runner.intermediate_output_ttl = 0
 654         runner.secret_store = cwltool.secrets.SecretStore()
 655         runner.label = mock.MagicMock()
 656         runner.label.return_value = '[container testjob]'
 657
 658         runner.api.containers().get().execute.return_value = {
 659             "state":"Complete",
 660             "output": "abc+123",
 661             "exit_code": 1,
 662             "log": "def+234"
 663         }
 664
 665         col().open.return_value = []
 666
 667         loadingContext, runtimeContext = self.helper(runner)
 668
 669         arvjob = arvados_cwl.ArvadosContainer(runner,
 670                                               runtimeContext,
 671                                               [],
 672                                               mock.MagicMock(),
 673                                               {},
 674                                               None,
 675                                               [],
 676                                               [],
 677                                               "testjob")
 678         arvjob.output_callback = mock.MagicMock()
 679         arvjob.collect_outputs = mock.MagicMock()
 680         arvjob.successCodes = [0]
 681         arvjob.outdir = "/var/spool/cwl"
 682         arvjob.output_ttl = 3600
 683         arvjob.collect_outputs.return_value = {"out": "stuff"}
 684
 685         arvjob.done({
 686             "state": "Final",
 687             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 688             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 689             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 690             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 691             "modified_at": "2017-05-26T12:01:22Z",
 692             "properties": {}
 693         })
 694
 695         rts_mock.assert_has_calls([
 696             mock.call('error',
 697                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 698                       '  ** log is empty **'
 699                       ),
 700             mock.call('warning',
 701                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 702         )])
 703         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 704
 705     # The test passes no builder.resources
 706     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 707     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 708     def test_mounts(self, keepdocker):
 709         runner = mock.MagicMock()
 710         runner.ignore_docker_for_reuse = False
 711         runner.intermediate_output_ttl = 0
 712         runner.secret_store = cwltool.secrets.SecretStore()
 713         runner.api._rootDesc = {"revision": "20210628"}
 714         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 715
 716         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 717         runner.api.collections().get().execute.return_value = {
 718             "portable_data_hash": "99999999999999999999999999999994+99",
 719             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 720
 721         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 722
 723         tool = cmap({
 724             "inputs": [
 725                 {"id": "p1",
 726                  "type": "Directory"}
 727             ],
 728             "outputs": [],
 729             "baseCommand": "ls",
 730             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 731             "id": "",
 732             "class": "CommandLineTool",
 733             "cwlVersion": "v1.2"
 734         })
 735
 736         loadingContext, runtimeContext = self.helper(runner)
 737         runtimeContext.name = "test_run_mounts"
 738
 739         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 740         arvtool.formatgraph = None
 741         job_order = {
 742             "p1": {
 743                 "class": "Directory",
 744                 "location": "keep:99999999999999999999999999999994+44",
 745                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 746                 "listing": [
 747                     {
 748                         "class": "File",
 749                         "location": "keep:99999999999999999999999999999994+44/file1",
 750                     },
 751                     {
 752                         "class": "File",
 753                         "location": "keep:99999999999999999999999999999994+44/file2",
 754                     }
 755                 ]
 756             }
 757         }
 758         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 759             j.run(runtimeContext)
 760             runner.api.container_requests().create.assert_called_with(
 761                 body=JsonDiffMatcher({
 762                     'environment': {
 763                         'HOME': '/var/spool/cwl',
 764                         'TMPDIR': '/tmp'
 765                     },
 766                     'name': 'test_run_mounts',
 767                     'runtime_constraints': {
 768                         'vcpus': 1,
 769                         'ram': 268435456
 770                     },
 771                     'use_existing': True,
 772                     'priority': 500,
 773                     'mounts': {
 774                         "/keep/99999999999999999999999999999994+44": {
 775                             "kind": "collection",
 776                             "portable_data_hash": "99999999999999999999999999999994+44",
 777                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 778                         },
 779                         '/tmp': {'kind': 'tmp',
 780                                  "capacity": 1073741824 },
 781                         '/var/spool/cwl': {'kind': 'tmp',
 782                                            "capacity": 1073741824 }
 783                     },
 784                     'state': 'Committed',
 785                     'output_name': 'Output from step test_run_mounts',
 786                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 787                     'output_path': '/var/spool/cwl',
 788                     'output_ttl': 0,
 789                     'container_image': '99999999999999999999999999999994+99',
 790                     'command': ['ls', '/var/spool/cwl'],
 791                     'cwd': '/var/spool/cwl',
 792                     'scheduling_parameters': {},
 793                     'properties': {'cwl_input': {
 794                         "p1": {
 795                             "basename": "99999999999999999999999999999994+44",
 796                             "class": "Directory",
 797                             "dirname": "/keep",
 798                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 799                             "listing": [
 800                                 {
 801                                     "basename": "file1",
 802                                     "class": "File",
 803                                     "dirname": "/keep/99999999999999999999999999999994+44",
 804                                     "location": "keep:99999999999999999999999999999994+44/file1",
 805                                     "nameext": "",
 806                                     "nameroot": "file1",
 807                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 808                                     "size": 0
 809                                 },
 810                                 {
 811                                     "basename": "file2",
 812                                     "class": "File",
 813                                     "dirname": "/keep/99999999999999999999999999999994+44",
 814                                     "location": "keep:99999999999999999999999999999994+44/file2",
 815                                     "nameext": "",
 816                                     "nameroot": "file2",
 817                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 818                                     "size": 0
 819                                 }
 820                             ],
 821                             "location": "keep:99999999999999999999999999999994+44",
 822                             "path": "/keep/99999999999999999999999999999994+44"
 823                         }
 824                     }},
 825                     'secret_mounts': {},
 826                     'output_storage_classes': ["default"]
 827                 }))
 828
 829     # The test passes no builder.resources
 830     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 831     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 832     def test_secrets(self, keepdocker):
 833         arvados_cwl.add_arv_hints()
 834         runner = mock.MagicMock()
 835         runner.ignore_docker_for_reuse = False
 836         runner.intermediate_output_ttl = 0
 837         runner.secret_store = cwltool.secrets.SecretStore()
 838         runner.api._rootDesc = {"revision": "20210628"}
 839         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 840
 841         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 842         runner.api.collections().get().execute.return_value = {
 843             "portable_data_hash": "99999999999999999999999999999993+99"}
 844
 845         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 846
 847         tool = cmap({"arguments": ["md5sum", "example.conf"],
 848                      "class": "CommandLineTool",
 849                      "cwlVersion": "v1.2",
 850                      "hints": [
 851                          {
 852                              "class": "http://commonwl.org/cwltool#Secrets",
 853                              "secrets": [
 854                                  "#secret_job.cwl/pw"
 855                              ]
 856                          }
 857                      ],
 858                      "id": "",
 859                      "inputs": [
 860                          {
 861                              "id": "#secret_job.cwl/pw",
 862                              "type": "string"
 863                          }
 864                      ],
 865                      "outputs": [
 866                      ],
 867                      "requirements": [
 868                          {
 869                              "class": "InitialWorkDirRequirement",
 870                              "listing": [
 871                                  {
 872                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 873                                      "entryname": "example.conf"
 874                                  }
 875                              ]
 876                          }
 877                      ]})
 878
 879         loadingContext, runtimeContext = self.helper(runner)
 880         runtimeContext.name = "test_secrets"
 881
 882         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 883         arvtool.formatgraph = None
 884
 885         job_order = {"pw": "blorp"}
 886         runner.secret_store.store(["pw"], job_order)
 887
 888         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 889             j.run(runtimeContext)
 890             runner.api.container_requests().create.assert_called_with(
 891                 body=JsonDiffMatcher({
 892                     'environment': {
 893                         'HOME': '/var/spool/cwl',
 894                         'TMPDIR': '/tmp'
 895                     },
 896                     'name': 'test_secrets',
 897                     'runtime_constraints': {
 898                         'vcpus': 1,
 899                         'ram': 268435456
 900                     },
 901                     'use_existing': True,
 902                     'priority': 500,
 903                     'mounts': {
 904                         '/tmp': {'kind': 'tmp',
 905                                  "capacity": 1073741824
 906                              },
 907                         '/var/spool/cwl': {'kind': 'tmp',
 908                                            "capacity": 1073741824 }
 909                     },
 910                     'state': 'Committed',
 911                     'output_name': 'Output from step test_secrets',
 912                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 913                     'output_path': '/var/spool/cwl',
 914                     'output_ttl': 0,
 915                     'container_image': '99999999999999999999999999999993+99',
 916                     'command': ['md5sum', 'example.conf'],
 917                     'cwd': '/var/spool/cwl',
 918                     'scheduling_parameters': {},
 919                     'properties': {'cwl_input': job_order},
 920                     "secret_mounts": {
 921                         "/var/spool/cwl/example.conf": {
 922                             "content": "username: user\npassword: blorp\n",
 923                             "kind": "text"
 924                         }
 925                     },
 926                     'output_storage_classes': ["default"]
 927                 }))
 928
 929     # The test passes no builder.resources
 930     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 931     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 932     def test_timelimit(self, keepdocker):
 933         runner = mock.MagicMock()
 934         runner.ignore_docker_for_reuse = False
 935         runner.intermediate_output_ttl = 0
 936         runner.secret_store = cwltool.secrets.SecretStore()
 937         runner.api._rootDesc = {"revision": "20210628"}
 938         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 939
 940         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 941         runner.api.collections().get().execute.return_value = {
 942             "portable_data_hash": "99999999999999999999999999999993+99"}
 943
 944         tool = cmap({
 945             "inputs": [],
 946             "outputs": [],
 947             "baseCommand": "ls",
 948             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 949             "id": "",
 950             "cwlVersion": "v1.2",
 951             "class": "CommandLineTool",
 952             "hints": [
 953                 {
 954                     "class": "ToolTimeLimit",
 955                     "timelimit": 42
 956                 }
 957             ]
 958         })
 959
 960         loadingContext, runtimeContext = self.helper(runner)
 961         runtimeContext.name = "test_timelimit"
 962
 963         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 964         arvtool.formatgraph = None
 965
 966         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 967             j.run(runtimeContext)
 968
 969         _, kwargs = runner.api.container_requests().create.call_args
 970         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 971
 972
 973     # The test passes no builder.resources
 974     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 975     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 976     def test_setting_storage_class(self, keepdocker):
 977         runner = mock.MagicMock()
 978         runner.ignore_docker_for_reuse = False
 979         runner.intermediate_output_ttl = 0
 980         runner.secret_store = cwltool.secrets.SecretStore()
 981         runner.api._rootDesc = {"revision": "20210628"}
 982         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 983
 984         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 985         runner.api.collections().get().execute.return_value = {
 986             "portable_data_hash": "99999999999999999999999999999993+99"}
 987
 988         tool = cmap({
 989             "inputs": [],
 990             "outputs": [],
 991             "baseCommand": "ls",
 992             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 993             "id": "",
 994             "cwlVersion": "v1.2",
 995             "class": "CommandLineTool",
 996             "hints": [
 997                 {
 998                     "class": "http://arvados.org/cwl#OutputStorageClass",
 999                     "finalStorageClass": ["baz_sc", "qux_sc"],
1000                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1001                 }
1002             ]
1003         })
1004
1005         loadingContext, runtimeContext = self.helper(runner, True)
1006
1007         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1008         arvtool.formatgraph = None
1009
1010         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1011             j.run(runtimeContext)
1012             runner.api.container_requests().create.assert_called_with(
1013                 body=JsonDiffMatcher({
1014                     'environment': {
1015                         'HOME': '/var/spool/cwl',
1016                         'TMPDIR': '/tmp'
1017                     },
1018                     'name': 'test_run_True',
1019                     'runtime_constraints': {
1020                         'vcpus': 1,
1021                         'ram': 268435456
1022                     },
1023                     'use_existing': True,
1024                     'priority': 500,
1025                     'mounts': {
1026                         '/tmp': {'kind': 'tmp',
1027                                  "capacity": 1073741824
1028                              },
1029                         '/var/spool/cwl': {'kind': 'tmp',
1030                                            "capacity": 1073741824 }
1031                     },
1032                     'state': 'Committed',
1033                     'output_name': 'Output from step test_run_True',
1034                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1035                     'output_path': '/var/spool/cwl',
1036                     'output_ttl': 0,
1037                     'container_image': '99999999999999999999999999999993+99',
1038                     'command': ['ls', '/var/spool/cwl'],
1039                     'cwd': '/var/spool/cwl',
1040                     'scheduling_parameters': {},
1041                     'properties': {'cwl_input': {}},
1042                     'secret_mounts': {},
1043                     'output_storage_classes': ["foo_sc", "bar_sc"]
1044                 }))
1045
1046
1047     # The test passes no builder.resources
1048     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1049     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1050     def test_setting_process_properties(self, keepdocker):
1051         runner = mock.MagicMock()
1052         runner.ignore_docker_for_reuse = False
1053         runner.intermediate_output_ttl = 0
1054         runner.secret_store = cwltool.secrets.SecretStore()
1055         runner.api._rootDesc = {"revision": "20210628"}
1056         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1057
1058         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1059         runner.api.collections().get().execute.return_value = {
1060             "portable_data_hash": "99999999999999999999999999999993+99"}
1061
1062         tool = cmap({
1063             "inputs": [
1064                 {"id": "x", "type": "string"}],
1065             "outputs": [],
1066             "baseCommand": "ls",
1067             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1068             "id": "",
1069             "class": "CommandLineTool",
1070             "cwlVersion": "v1.2",
1071             "hints": [
1072             {
1073                 "class": "http://arvados.org/cwl#ProcessProperties",
1074                 "processProperties": [
1075                     {"propertyName": "foo",
1076                      "propertyValue": "bar"},
1077                     {"propertyName": "baz",
1078                      "propertyValue": "$(inputs.x)"},
1079                     {"propertyName": "quux",
1080                      "propertyValue": {
1081                          "q1": 1,
1082                          "q2": 2
1083                      }
1084                     }
1085                 ],
1086             }
1087         ]
1088         })
1089
1090         loadingContext, runtimeContext = self.helper(runner, True)
1091
1092         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1093         arvtool.formatgraph = None
1094
1095         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1096             j.run(runtimeContext)
1097             runner.api.container_requests().create.assert_called_with(
1098                 body=JsonDiffMatcher({
1099                     'environment': {
1100                         'HOME': '/var/spool/cwl',
1101                         'TMPDIR': '/tmp'
1102                     },
1103                     'name': 'test_run_True',
1104                     'runtime_constraints': {
1105                         'vcpus': 1,
1106                         'ram': 268435456
1107                     },
1108                     'use_existing': True,
1109                     'priority': 500,
1110                     'mounts': {
1111                         '/tmp': {'kind': 'tmp',
1112                                  "capacity": 1073741824
1113                              },
1114                         '/var/spool/cwl': {'kind': 'tmp',
1115                                            "capacity": 1073741824 }
1116                     },
1117                     'state': 'Committed',
1118                     'output_name': 'Output from step test_run_True',
1119                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1120                     'output_path': '/var/spool/cwl',
1121                     'output_ttl': 0,
1122                     'container_image': '99999999999999999999999999999993+99',
1123                     'command': ['ls', '/var/spool/cwl'],
1124                     'cwd': '/var/spool/cwl',
1125                     'scheduling_parameters': {},
1126                     'properties': {
1127                         "baz": "blorp",
1128                         "cwl_input": {"x": "blorp"},
1129                         "foo": "bar",
1130                         "quux": {
1131                             "q1": 1,
1132                             "q2": 2
1133                         }
1134                     },
1135                     'secret_mounts': {},
1136                     'output_storage_classes': ["default"]
1137                 }))
1138
1139
1140     # The test passes no builder.resources
1141     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1142     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1143     def test_cuda_requirement(self, keepdocker):
1144         arvados_cwl.add_arv_hints()
1145
1146         runner = mock.MagicMock()
1147         runner.ignore_docker_for_reuse = False
1148         runner.intermediate_output_ttl = 0
1149         runner.secret_store = cwltool.secrets.SecretStore()
1150         runner.api._rootDesc = {"revision": "20210628"}
1151         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1152
1153         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1154         runner.api.collections().get().execute.return_value = {
1155             "portable_data_hash": "99999999999999999999999999999993+99"}
1156
1157         test_cwl_req = [{
1158                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1159                 "cudaVersionMin": "11.0",
1160                 "cudaComputeCapability": "9.0",
1161             }, {
1162                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1163                 "cudaVersionMin": "11.0",
1164                 "cudaComputeCapability": "9.0",
1165                 "cudaDeviceCountMin": 2
1166             }, {
1167                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1168                 "cudaVersionMin": "11.0",
1169                 "cudaComputeCapability": ["4.0", "5.0"],
1170                 "cudaDeviceCountMin": 2
1171             }]
1172
1173         test_arv_req = [{
1174             'device_count': 1,
1175             'driver_version': "11.0",
1176             'hardware_capability': "9.0"
1177         }, {
1178             'device_count': 2,
1179             'driver_version': "11.0",
1180             'hardware_capability': "9.0"
1181         }, {
1182             'device_count': 2,
1183             'driver_version': "11.0",
1184             'hardware_capability': "4.0"
1185         }]
1186
1187         for test_case in range(0, len(test_cwl_req)):
1188
1189             tool = cmap({
1190                 "inputs": [],
1191                 "outputs": [],
1192                 "baseCommand": "nvidia-smi",
1193                 "arguments": [],
1194                 "id": "",
1195                 "cwlVersion": "v1.2",
1196                 "class": "CommandLineTool",
1197                 "requirements": [test_cwl_req[test_case]]
1198             })
1199
1200             loadingContext, runtimeContext = self.helper(runner, True)
1201
1202             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1203             arvtool.formatgraph = None
1204
1205             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1206                 j.run(runtimeContext)
1207                 runner.api.container_requests().create.assert_called_with(
1208                     body=JsonDiffMatcher({
1209                         'environment': {
1210                             'HOME': '/var/spool/cwl',
1211                             'TMPDIR': '/tmp'
1212                         },
1213                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1214                         'runtime_constraints': {
1215                             'vcpus': 1,
1216                             'ram': 268435456,
1217                             'cuda': test_arv_req[test_case]
1218                         },
1219                         'use_existing': True,
1220                         'priority': 500,
1221                         'mounts': {
1222                             '/tmp': {'kind': 'tmp',
1223                                      "capacity": 1073741824
1224                                  },
1225                             '/var/spool/cwl': {'kind': 'tmp',
1226                                                "capacity": 1073741824 }
1227                         },
1228                         'state': 'Committed',
1229                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1230                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1231                         'output_path': '/var/spool/cwl',
1232                         'output_ttl': 0,
1233                         'container_image': '99999999999999999999999999999993+99',
1234                         'command': ['nvidia-smi'],
1235                         'cwd': '/var/spool/cwl',
1236                         'scheduling_parameters': {},
1237                         'properties': {'cwl_input': {}},
1238                         'secret_mounts': {},
1239                         'output_storage_classes': ["default"]
1240                     }))
1241
1242
1243     # The test passes no builder.resources
1244     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1245     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1246     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1247     def test_match_local_docker(self, keepdocker, determine_image_id):
1248         arvados_cwl.add_arv_hints()
1249
1250         runner = mock.MagicMock()
1251         runner.ignore_docker_for_reuse = False
1252         runner.intermediate_output_ttl = 0
1253         runner.secret_store = cwltool.secrets.SecretStore()
1254         runner.api._rootDesc = {"revision": "20210628"}
1255         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1256
1257         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1258                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1259         determine_image_id.side_effect = lambda x: "123"
1260         def execute(uuid):
1261             ex = mock.MagicMock()
1262             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1263                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1264             ex.execute.return_value = lookup[uuid]
1265             return ex
1266         runner.api.collections().get.side_effect = execute
1267
1268         tool = cmap({
1269             "inputs": [],
1270             "outputs": [],
1271             "baseCommand": "echo",
1272             "arguments": [],
1273             "id": "",
1274             "cwlVersion": "v1.0",
1275             "class": "org.w3id.cwl.cwl.CommandLineTool"
1276         })
1277
1278         loadingContext, runtimeContext = self.helper(runner, True)
1279
1280         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1281         arvtool.formatgraph = None
1282
1283         container_request = {
1284             'environment': {
1285                 'HOME': '/var/spool/cwl',
1286                 'TMPDIR': '/tmp'
1287             },
1288             'name': 'test_run_True',
1289             'runtime_constraints': {
1290                 'vcpus': 1,
1291                 'ram': 1073741824,
1292             },
1293             'use_existing': True,
1294             'priority': 500,
1295             'mounts': {
1296                 '/tmp': {'kind': 'tmp',
1297                          "capacity": 1073741824
1298                          },
1299                 '/var/spool/cwl': {'kind': 'tmp',
1300                                    "capacity": 1073741824 }
1301             },
1302             'state': 'Committed',
1303             'output_name': 'Output from step test_run_True',
1304             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1305             'output_path': '/var/spool/cwl',
1306             'output_ttl': 0,
1307             'container_image': '99999999999999999999999999999994+99',
1308             'command': ['echo'],
1309             'cwd': '/var/spool/cwl',
1310             'scheduling_parameters': {},
1311             'properties': {'cwl_input': {}},
1312             'secret_mounts': {},
1313             'output_storage_classes': ["default"]
1314         }
1315
1316         runtimeContext.match_local_docker = False
1317         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1318             j.run(runtimeContext)
1319             runner.api.container_requests().create.assert_called_with(
1320                 body=JsonDiffMatcher(container_request))
1321
1322         runtimeContext.cached_docker_lookups.clear()
1323         runtimeContext.match_local_docker = True
1324         container_request['container_image'] = '99999999999999999999999999999993+99'
1325         container_request['name'] = 'test_run_True_2'
1326         container_request['output_name'] = 'Output from step test_run_True_2'
1327         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1328             j.run(runtimeContext)
1329             runner.api.container_requests().create.assert_called_with(
1330                 body=JsonDiffMatcher(container_request))
1331
1332
1333     # The test passes no builder.resources
1334     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1335     @parameterized.expand([
1336         ("None, None",   None,  None,  None),
1337         ("None, True",   None,  True,  True),
1338         ("None, False",  None,  False, False),
1339         ("False, None",  False, None,  False),
1340         ("False, True",  False, True,  False),
1341         ("False, False", False, False, False),
1342         ("True, None",   True,  None,  True),
1343         ("True, True",   True,  True,  True),
1344         ("True, False",  True,  False, False),
1345     ])
1346     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1347     def test_run_preemptible_hint(self, _, enable_preemptible, preemptible_hint,
1348                                   preemptible_setting, keepdocker):
1349         arvados_cwl.add_arv_hints()
1350
1351         runner = mock.MagicMock()
1352         runner.ignore_docker_for_reuse = False
1353         runner.intermediate_output_ttl = 0
1354         runner.secret_store = cwltool.secrets.SecretStore()
1355         runner.api._rootDesc = {"revision": "20210628"}
1356         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1357
1358         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1359         runner.api.collections().get().execute.return_value = {
1360             "portable_data_hash": "99999999999999999999999999999993+99"}
1361
1362         if preemptible_hint is not None:
1363             hints = [{
1364                 "class": "http://arvados.org/cwl#UsePreemptible",
1365                 "usePreemptible": preemptible_hint
1366             }]
1367         else:
1368             hints = []
1369
1370         tool = cmap({
1371             "inputs": [],
1372             "outputs": [],
1373             "baseCommand": "ls",
1374             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1375             "id": "",
1376             "class": "CommandLineTool",
1377             "cwlVersion": "v1.2",
1378             "hints": hints
1379         })
1380
1381         loadingContext, runtimeContext = self.helper(runner)
1382
1383         runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1384         runtimeContext.enable_preemptible = enable_preemptible
1385
1386         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1387         arvtool.formatgraph = None
1388
1389         # Test the interactions between --enable/disable-preemptible
1390         # and UsePreemptible hint
1391
1392         sched = {}
1393         if preemptible_setting is not None:
1394             sched['preemptible'] = preemptible_setting
1395
1396         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1397             j.run(runtimeContext)
1398             runner.api.container_requests().create.assert_called_with(
1399                 body=JsonDiffMatcher({
1400                     'environment': {
1401                         'HOME': '/var/spool/cwl',
1402                         'TMPDIR': '/tmp'
1403                     },
1404                     'name': runtimeContext.name,
1405                     'runtime_constraints': {
1406                         'vcpus': 1,
1407                         'ram': 268435456
1408                     },
1409                     'use_existing': True,
1410                     'priority': 500,
1411                     'mounts': {
1412                         '/tmp': {'kind': 'tmp',
1413                                  "capacity": 1073741824
1414                              },
1415                         '/var/spool/cwl': {'kind': 'tmp',
1416                                            "capacity": 1073741824 }
1417                     },
1418                     'state': 'Committed',
1419                     'output_name': 'Output from step '+runtimeContext.name,
1420                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1421                     'output_path': '/var/spool/cwl',
1422                     'output_ttl': 0,
1423                     'container_image': '99999999999999999999999999999993+99',
1424                     'command': ['ls', '/var/spool/cwl'],
1425                     'cwd': '/var/spool/cwl',
1426                     'scheduling_parameters': sched,
1427                     'properties': {'cwl_input': {}},
1428                     'secret_mounts': {},
1429                     'output_storage_classes': ["default"]
1430                 }))
1431
1432
1433     # The test passes no builder.resources
1434     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1435     @parameterized.expand([
1436         ("None, None", None, None, False),
1437         ("None, True", None, True, True),
1438         ("None, False", None, False, False),
1439         ("False, None", False, None, False),
1440         ("False, True", False, True, False),  # command line overrides hint
1441         ("False, False", False, False, False),
1442         ("True, None", True, None, True),
1443         ("True, True", True, True, True),
1444         ("True, False", True, False, False),  # hint overrides command line
1445     ])
1446     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1447     def test_spot_retry(self, _, enable_resubmit_non_preemptible,
1448                         preemption_behavior_hint,
1449                         expect_resubmit_behavior,
1450                         keepdocker):
1451         arvados_cwl.add_arv_hints()
1452
1453         runner = mock.MagicMock()
1454         runner.ignore_docker_for_reuse = False
1455         runner.intermediate_output_ttl = 0
1456         runner.secret_store = cwltool.secrets.SecretStore()
1457         runner.api._rootDesc = {"revision": "20210628"}
1458         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1459
1460         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1461         runner.api.collections().get().execute.return_value = {
1462             "portable_data_hash": "99999999999999999999999999999993+99"}
1463
1464
1465         hints = [{
1466             "class": "http://arvados.org/cwl#UsePreemptible",
1467             "usePreemptible": True
1468         }]
1469
1470         if preemption_behavior_hint is not None:
1471             hints.append({
1472                 "class": "http://arvados.org/cwl#PreemptionBehavior",
1473                 "resubmitNonPreemptible": preemption_behavior_hint
1474             })
1475
1476         tool = cmap({
1477             "inputs": [],
1478             "outputs": [],
1479             "baseCommand": "ls",
1480             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1481             "id": "",
1482             "class": "CommandLineTool",
1483             "cwlVersion": "v1.2",
1484             "hints": hints
1485         })
1486
1487         loadingContext, runtimeContext = self.helper(runner)
1488
1489         runtimeContext.name = 'test_spot_retry_'+str(enable_resubmit_non_preemptible)+str(preemption_behavior_hint)
1490         runtimeContext.enable_resubmit_non_preemptible = enable_resubmit_non_preemptible
1491
1492         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1493         arvtool.formatgraph = None
1494
1495         # Test the interactions between --enable/disable-preemptible
1496         # and UsePreemptible hint
1497
1498         expect_container_request = {
1499                     'environment': {
1500                         'HOME': '/var/spool/cwl',
1501                         'TMPDIR': '/tmp'
1502                     },
1503                     'name': runtimeContext.name,
1504                     'runtime_constraints': {
1505                         'vcpus': 1,
1506                         'ram': 268435456
1507                     },
1508                     'use_existing': True,
1509                     'priority': 500,
1510                     'mounts': {
1511                         '/tmp': {'kind': 'tmp',
1512                                  "capacity": 1073741824
1513                              },
1514                         '/var/spool/cwl': {'kind': 'tmp',
1515                                            "capacity": 1073741824 }
1516                     },
1517                     'state': 'Committed',
1518                     'output_name': 'Output from step '+runtimeContext.name,
1519                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1520                     'output_path': '/var/spool/cwl',
1521                     'output_ttl': 0,
1522                     'container_image': '99999999999999999999999999999993+99',
1523                     'command': ['ls', '/var/spool/cwl'],
1524                     'cwd': '/var/spool/cwl',
1525                     'scheduling_parameters': {'preemptible': True},
1526                     'properties': {'cwl_input': {}},
1527                     'secret_mounts': {},
1528                     'output_storage_classes': ["default"],
1529                 }
1530
1531         expect_resubmit_container_request = expect_container_request.copy()
1532         expect_resubmit_container_request['scheduling_parameters'] = {'preemptible': False}
1533
1534         runner.api.container_requests().create().execute.return_value = {"uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
1535                                                                          "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
1536
1537         if expect_resubmit_behavior:
1538             expect_container_request['container_count_max'] = 1
1539
1540         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1541             j.run(runtimeContext)
1542             runner.api.container_requests().create.assert_called_with(
1543                 body=JsonDiffMatcher(expect_container_request))
1544         runner.api.containers().get().execute.return_value = {
1545             "state":"Cancelled",
1546             "output": "abc+123",
1547             "exit_code": 1,
1548             "log": "def+234",
1549             "runtime_status": {
1550                 "preemptionNotice": "bye bye"
1551             }
1552         }
1553         runner.api.container_requests().create().execute.return_value = {"uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzz2",
1554                                                                          "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzz2"}
1555
1556         j.done({
1557             "state": "Final",
1558             "log_uuid": "",
1559             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
1560             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
1561             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
1562             "modified_at": "2017-05-26T12:01:22Z",
1563             "properties": {},
1564             "name": "testjob"
1565         })
1566         if expect_resubmit_behavior:
1567             runner.api.container_requests().update.assert_any_call(
1568                 uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz", body={"properties": {"arv:failed_container_resubmitted": "zzzzz-xvhdp-zzzzzzzzzzzzzz2"}})
1569             runner.api.container_requests().create.assert_called_with(
1570                 body=JsonDiffMatcher(expect_resubmit_container_request))
1571
1572     @parameterized.expand([
1573         ("20210628",),
1574         ("20220510",),
1575     ])
1576     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1577     def test_output_properties(self, rev, keepdocker):
1578         arvados_cwl.add_arv_hints()
1579         runner = mock.MagicMock()
1580         runner.ignore_docker_for_reuse = False
1581         runner.intermediate_output_ttl = 0
1582         runner.secret_store = cwltool.secrets.SecretStore()
1583         runner.api._rootDesc = {"revision": rev}
1584         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1585
1586         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1587         runner.api.collections().get().execute.return_value = {
1588             "portable_data_hash": "99999999999999999999999999999993+99"}
1589
1590         tool = cmap({
1591             "inputs": [{
1592                 "id": "inp",
1593                 "type": "string"
1594             }],
1595             "outputs": [],
1596             "baseCommand": "ls",
1597             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1598             "id": "",
1599             "cwlVersion": "v1.2",
1600             "class": "CommandLineTool",
1601             "hints": [
1602                 {
1603                     "class": "http://arvados.org/cwl#OutputCollectionProperties",
1604                     "outputProperties": {
1605                         "foo": "bar",
1606                         "baz": "$(inputs.inp)"
1607                     }
1608                 }
1609             ]
1610         })
1611
1612         loadingContext, runtimeContext = self.helper(runner)
1613         runtimeContext.name = "test_timelimit"
1614
1615         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1616         arvtool.formatgraph = None
1617
1618         for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1619             j.run(runtimeContext)
1620
1621         _, kwargs = runner.api.container_requests().create.call_args
1622         if rev == "20220510":
1623             self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1624         else:
1625             self.assertEqual(None, kwargs['body'].get('output_properties'))
1626
1627     @parameterized.expand([
1628         ("20231117",),
1629         ("20240502",),
1630     ])
1631     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1632     def test_output_glob(self, rev, keepdocker):
1633         arvados_cwl.add_arv_hints()
1634         runner = mock.MagicMock()
1635         runner.ignore_docker_for_reuse = False
1636         runner.intermediate_output_ttl = 0
1637         runner.secret_store = cwltool.secrets.SecretStore()
1638         runner.api._rootDesc = {"revision": rev}
1639         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1640
1641         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1642         runner.api.collections().get().execute.return_value = {
1643             "portable_data_hash": "99999999999999999999999999999993+99"}
1644
1645         tool = cmap({
1646             "inputs": [{
1647                 "id": "inp",
1648                 "type": "string"
1649             }],
1650             "outputs": [
1651                 {
1652                     "id": "o1",
1653                     "type": "File",
1654                     "outputBinding": {
1655                         "glob": "*.txt"
1656                     }
1657                 },
1658                 {
1659                     "id": "o2",
1660                     "type": "File",
1661                     "outputBinding": {
1662                         "glob": ["*.dat", "*.bat"]
1663                     }
1664                 },
1665                 {
1666                     "id": "o3",
1667                     "type": {
1668                         "type": "record",
1669                         "fields": [
1670                             {
1671                                 "name": "f1",
1672                                 "type": "File",
1673                                 "outputBinding": {
1674                                     "glob": ["*.cat"]
1675                                 }
1676                             }
1677                         ]
1678                     }
1679                 },
1680                 {
1681                     "id": "o4",
1682                     "type": "File",
1683                     "outputBinding": {
1684                         "glob": "$(inputs.inp)"
1685                     }
1686                 },
1687                 {
1688                     "id": "o5",
1689                     "type": "File",
1690                     "outputBinding": {
1691                         "glob": "*.foo"
1692                     },
1693                     "secondaryFiles": [".goo", "^.hoo"]
1694                 },
1695
1696             ],
1697             "baseCommand": "ls",
1698             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1699             "id": "",
1700             "cwlVersion": "v1.2",
1701             "class": "CommandLineTool",
1702             "hints": [ ]
1703         })
1704
1705         loadingContext, runtimeContext = self.helper(runner)
1706         runtimeContext.name = "test_timelimit"
1707
1708         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1709         arvtool.formatgraph = None
1710
1711         for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1712             j.run(runtimeContext)
1713
1714         _, kwargs = runner.api.container_requests().create.call_args
1715         if rev == "20240502":
1716             self.assertEqual(['*.txt', '*.txt/**',
1717                               '*.dat', '*.dat/**',
1718                               '*.bat', '*.bat/**',
1719                               '*.cat', '*.cat/**',
1720                               'quux', 'quux/**',
1721                               '*.foo', '*.foo/**',
1722                               '*.foo.goo', '*.foo.goo/**',
1723                               '*.hoo', '*.hoo/**',
1724                               'cwl.output.json',
1725                               ], kwargs['body'].get('output_glob'))
1726         else:
1727             self.assertEqual(None, kwargs['body'].get('output_glob'))
1728
1729
1730 class TestWorkflow(unittest.TestCase):
1731     def setUp(self):
1732         cwltool.process._names = set()
1733
1734     def helper(self, runner, enable_reuse=True):
1735         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1736
1737         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1738                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1739
1740         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1741         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1742         document_loader.fetch_text = document_loader.fetcher.fetch_text
1743         document_loader.check_exists = document_loader.fetcher.check_exists
1744
1745         loadingContext = arvados_cwl.context.ArvLoadingContext(
1746             {"avsc_names": avsc_names,
1747              "basedir": "",
1748              "make_fs_access": make_fs_access,
1749              "loader": document_loader,
1750              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1751              "construct_tool_object": runner.arv_make_tool,
1752              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__,
1753              })
1754         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1755             {"work_api": "containers",
1756              "basedir": "",
1757              "name": "test_run_wf_"+str(enable_reuse),
1758              "make_fs_access": make_fs_access,
1759              "tmpdir": "/tmp",
1760              "enable_reuse": enable_reuse,
1761              "priority": 500})
1762
1763         return loadingContext, runtimeContext
1764
1765     # The test passes no builder.resources
1766     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1767     @mock.patch("arvados.collection.CollectionReader")
1768     @mock.patch("arvados.collection.Collection")
1769     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1770     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1771         arvados_cwl.add_arv_hints()
1772
1773         api = mock.MagicMock()
1774         api._rootDesc = get_rootDesc()
1775         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1776
1777         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1778         self.assertEqual(runner.work_api, 'containers')
1779
1780         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1781         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1782         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1783                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1784
1785         runner.api.containers().current().execute.return_value = {}
1786
1787         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1788         runner.ignore_docker_for_reuse = False
1789         runner.num_retries = 0
1790         runner.secret_store = cwltool.secrets.SecretStore()
1791
1792         loadingContext, runtimeContext = self.helper(runner)
1793         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1794
1795         mockcollectionreader().exists.return_value = True
1796
1797         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1798         metadata["cwlVersion"] = tool["cwlVersion"]
1799
1800         mockc = mock.MagicMock()
1801         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1802         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1803
1804         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1805         arvtool.formatgraph = None
1806         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1807
1808         next(it).run(runtimeContext)
1809         next(it).run(runtimeContext)
1810
1811         with open("tests/wf/scatter2_subwf.cwl") as f:
1812             subwf = StripYAMLComments(f.read()).rstrip()
1813
1814         runner.api.container_requests().create.assert_called_with(
1815             body=JsonDiffMatcher({
1816                 "command": [
1817                     "cwltool",
1818                     "--no-container",
1819                     "--move-outputs",
1820                     "--preserve-entire-environment",
1821                     "workflow.cwl",
1822                     "cwl.input.yml"
1823                 ],
1824                 "container_image": "99999999999999999999999999999993+99",
1825                 "cwd": "/var/spool/cwl",
1826                 "environment": {
1827                     "HOME": "/var/spool/cwl",
1828                     "TMPDIR": "/tmp"
1829                 },
1830                 "mounts": {
1831                     "/keep/99999999999999999999999999999999+118": {
1832                         "kind": "collection",
1833                         "portable_data_hash": "99999999999999999999999999999999+118"
1834                     },
1835                     "/tmp": {
1836                         "capacity": 1073741824,
1837                         "kind": "tmp"
1838                     },
1839                     "/var/spool/cwl": {
1840                         "capacity": 1073741824,
1841                         "kind": "tmp"
1842                     },
1843                     "/var/spool/cwl/cwl.input.yml": {
1844                         "kind": "collection",
1845                         "path": "cwl.input.yml",
1846                         "portable_data_hash": "99999999999999999999999999999996+99"
1847                     },
1848                     "/var/spool/cwl/workflow.cwl": {
1849                         "kind": "collection",
1850                         "path": "workflow.cwl",
1851                         "portable_data_hash": "99999999999999999999999999999996+99"
1852                     },
1853                     "stdout": {
1854                         "kind": "file",
1855                         "path": "/var/spool/cwl/cwl.output.json"
1856                     }
1857                 },
1858                 "name": "scatterstep",
1859                 "output_name": "Output from step scatterstep",
1860                 "output_path": "/var/spool/cwl",
1861                 "output_ttl": 0,
1862                 "priority": 500,
1863                 "properties": {'cwl_input': {
1864                         "fileblub": {
1865                             "basename": "token.txt",
1866                             "class": "File",
1867                             "dirname": "/keep/99999999999999999999999999999999+118",
1868                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1869                             "nameext": ".txt",
1870                             "nameroot": "token",
1871                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1872                             "size": 0
1873                         },
1874                         "sleeptime": 5
1875                 }},
1876                 "runtime_constraints": {
1877                     "ram": 1073741824,
1878                     "vcpus": 1
1879                 },
1880                 "scheduling_parameters": {},
1881                 "secret_mounts": {},
1882                 "state": "Committed",
1883                 "use_existing": True,
1884                 'output_storage_classes': ["default"]
1885             }))
1886         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1887         mockc.open().__enter__().write.assert_has_calls([mock.call(
1888 '''{
1889   "fileblub": {
1890     "basename": "token.txt",
1891     "class": "File",
1892     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1893     "size": 0
1894   },
1895   "sleeptime": 5
1896 }''')])
1897
1898     # The test passes no builder.resources
1899     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1900     @mock.patch("arvados.collection.CollectionReader")
1901     @mock.patch("arvados.collection.Collection")
1902     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1903     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1904         arvados_cwl.add_arv_hints()
1905
1906         api = mock.MagicMock()
1907         api._rootDesc = get_rootDesc()
1908         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1909
1910         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1911         self.assertEqual(runner.work_api, 'containers')
1912
1913         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1914         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1915                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1916         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1917                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1918
1919         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1920         runner.ignore_docker_for_reuse = False
1921         runner.num_retries = 0
1922         runner.secret_store = cwltool.secrets.SecretStore()
1923
1924         loadingContext, runtimeContext = self.helper(runner)
1925         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1926         loadingContext.do_update = True
1927         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1928
1929         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1930
1931         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1932         arvtool.formatgraph = None
1933         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1934
1935         next(it).run(runtimeContext)
1936         next(it).run(runtimeContext)
1937
1938         with open("tests/wf/echo-subwf.cwl") as f:
1939             subwf = StripYAMLComments(f.read())
1940
1941         runner.api.container_requests().create.assert_called_with(
1942             body=JsonDiffMatcher({
1943                 'output_ttl': 0,
1944                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1945                 'scheduling_parameters': {},
1946                 'name': u'echo-subwf',
1947                 'secret_mounts': {},
1948                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1949                 'properties': {'cwl_input': {}},
1950                 'priority': 500,
1951                 'mounts': {
1952                     '/var/spool/cwl/cwl.input.yml': {
1953                         'portable_data_hash': '99999999999999999999999999999996+99',
1954                         'kind': 'collection',
1955                         'path': 'cwl.input.yml'
1956                     },
1957                     '/var/spool/cwl/workflow.cwl': {
1958                         'portable_data_hash': '99999999999999999999999999999996+99',
1959                         'kind': 'collection',
1960                         'path': 'workflow.cwl'
1961                     },
1962                     'stdout': {
1963                         'path': '/var/spool/cwl/cwl.output.json',
1964                         'kind': 'file'
1965                     },
1966                     '/tmp': {
1967                         'kind': 'tmp',
1968                         'capacity': 1073741824
1969                     }, '/var/spool/cwl': {
1970                         'kind': 'tmp',
1971                         'capacity': 3221225472
1972                     }
1973                 },
1974                 'state': 'Committed',
1975                 'output_path': '/var/spool/cwl',
1976                 'container_image': '99999999999999999999999999999993+99',
1977                 'command': [
1978                     u'cwltool',
1979                     u'--no-container',
1980                     u'--move-outputs',
1981                     u'--preserve-entire-environment',
1982                     u'workflow.cwl',
1983                     u'cwl.input.yml'
1984                 ],
1985                 'use_existing': True,
1986                 'output_name': u'Output from step echo-subwf',
1987                 'cwd': '/var/spool/cwl',
1988                 'output_storage_classes': ["default"]
1989             }))
1990
1991     def test_default_work_api(self):
1992         arvados_cwl.add_arv_hints()
1993
1994         api = mock.MagicMock()
1995         api._rootDesc = copy.deepcopy(get_rootDesc())
1996         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1997         self.assertEqual(runner.work_api, 'containers')