sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 import arvados_cwl.util
   8 #from arvados_cwl.arvdocker import arv_docker_clear_cache
   9 import copy
  10 import arvados.config
  11 import logging
  12 import unittest
  13 import os
  14 import functools
  15 import threading
  16 import cwltool.process
  17 import cwltool.secrets
  18 import cwltool.load_tool
  19 from cwltool.update import INTERNAL_VERSION
  20 from schema_salad.ref_resolver import Loader
  21 from schema_salad.sourceline import cmap
  22 import io
  23 from parameterized import parameterized
  24
  25 from unittest import mock
  26
  27 from .matcher import JsonDiffMatcher, StripYAMLComments
  28 from .mock_discovery import get_rootDesc
  29
  30 if not os.getenv('ARVADOS_DEBUG'):
  31     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  32     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  33
  34 class CollectionMock(object):
  35     def __init__(self, vwdmock, *args, **kwargs):
  36         self.vwdmock = vwdmock
  37         self.count = 0
  38
  39     def open(self, *args, **kwargs):
  40         self.count += 1
  41         return self.vwdmock.open(*args, **kwargs)
  42
  43     def copy(self, *args, **kwargs):
  44         self.count += 1
  45         self.vwdmock.copy(*args, **kwargs)
  46
  47     def save_new(self, *args, **kwargs):
  48         pass
  49
  50     def __len__(self):
  51         return self.count
  52
  53     def portable_data_hash(self):
  54         if self.count == 0:
  55             return arvados.config.EMPTY_BLOCK_LOCATOR
  56         else:
  57             return "99999999999999999999999999999996+99"
  58
  59
  60 class TestContainer(unittest.TestCase):
  61
  62     def setUp(self):
  63         cwltool.process._names = set()
  64         #arv_docker_clear_cache()
  65
  66     def tearDown(self):
  67         root_logger = logging.getLogger('')
  68
  69         # Remove existing RuntimeStatusLoggingHandlers if they exist
  70         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  71         root_logger.handlers = handlers
  72
  73     def helper(self, runner, enable_reuse=True):
  74         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  75
  76         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  77                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  78         fs_access = mock.MagicMock()
  79         fs_access.exists.return_value = True
  80
  81         loadingContext = arvados_cwl.context.ArvLoadingContext(
  82             {"avsc_names": avsc_names,
  83              "basedir": "",
  84              "make_fs_access": make_fs_access,
  85              "construct_tool_object": runner.arv_make_tool,
  86              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  87              "loader": Loader({}),
  88              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  89              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__,
  90              })
  91         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  92             {"work_api": "containers",
  93              "basedir": "",
  94              "name": "test_run_"+str(enable_reuse),
  95              "make_fs_access": make_fs_access,
  96              "tmpdir": "/tmp",
  97              "outdir": "/tmp",
  98              "enable_reuse": enable_reuse,
  99              "priority": 500,
 100              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 101              "workflow_eval_lock": threading.Condition(threading.RLock())
 102             })
 103
 104         if isinstance(runner, mock.MagicMock):
 105             def make_tool(toolpath_object, loadingContext):
 106                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 107             runner.arv_make_tool.side_effect = make_tool
 108
 109         return loadingContext, runtimeContext
 110
 111     # Helper function to set up the ArvCwlExecutor to use the containers api
 112     # and test that the RuntimeStatusLoggingHandler is set up correctly
 113     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 114         api = mock.MagicMock()
 115         api._rootDesc = copy.deepcopy(get_rootDesc())
 116
 117         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 118         # adds the logging handler that will call runtime_status_update() mock
 119         self.assertFalse(gcc_mock.called)
 120         runner = arvados_cwl.ArvCwlExecutor(api)
 121         self.assertEqual(runner.work_api, 'containers')
 122         root_logger = logging.getLogger('')
 123         handlerClasses = [h.__class__ for h in root_logger.handlers]
 124         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 125         return runner
 126
 127     # The test passes no builder.resources
 128     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 129     @parameterized.expand([
 130         (True,),
 131         (False,),
 132     ])
 133     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 134     def test_run(self, enable_reuse, keepdocker):
 135         runner = mock.MagicMock()
 136         runner.ignore_docker_for_reuse = False
 137         runner.intermediate_output_ttl = 0
 138         runner.secret_store = cwltool.secrets.SecretStore()
 139         runner.api._rootDesc = {"revision": "20210628"}
 140         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 141
 142         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 143         runner.api.collections().get().execute.return_value = {
 144             "portable_data_hash": "99999999999999999999999999999993+99"}
 145
 146         tool = cmap({
 147             "inputs": [],
 148             "outputs": [],
 149             "baseCommand": "ls",
 150             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 151             "id": "",
 152             "class": "CommandLineTool",
 153             "cwlVersion": "v1.2"
 154         })
 155
 156         loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 157
 158         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 159         arvtool.formatgraph = None
 160
 161         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 162             j.run(runtimeContext)
 163             runner.api.container_requests().create.assert_called_with(
 164                 body=JsonDiffMatcher({
 165                     'environment': {
 166                         'HOME': '/var/spool/cwl',
 167                         'TMPDIR': '/tmp'
 168                     },
 169                     'name': 'test_run_'+str(enable_reuse),
 170                     'runtime_constraints': {
 171                         'vcpus': 1,
 172                         'ram': 268435456
 173                     },
 174                     'use_existing': enable_reuse,
 175                     'priority': 500,
 176                     'mounts': {
 177                         '/tmp': {'kind': 'tmp',
 178                                  "capacity": 1073741824
 179                              },
 180                         '/var/spool/cwl': {'kind': 'tmp',
 181                                            "capacity": 1073741824 }
 182                     },
 183                     'state': 'Committed',
 184                     'output_name': 'Output from step test_run_'+str(enable_reuse),
 185                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 186                     'output_path': '/var/spool/cwl',
 187                     'output_ttl': 0,
 188                     'container_image': '99999999999999999999999999999993+99',
 189                     'command': ['ls', '/var/spool/cwl'],
 190                     'cwd': '/var/spool/cwl',
 191                     'scheduling_parameters': {},
 192                     'properties': {'cwl_input': {}},
 193                     'secret_mounts': {},
 194                     'output_storage_classes': ["default"]
 195                 }))
 196
 197     # The test passes some fields in builder.resources
 198     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 199     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 200     def test_resource_requirements(self, keepdocker):
 201         arvados_cwl.add_arv_hints()
 202         runner = mock.MagicMock()
 203         runner.ignore_docker_for_reuse = False
 204         runner.intermediate_output_ttl = 3600
 205         runner.secret_store = cwltool.secrets.SecretStore()
 206         runner.api._rootDesc = {"revision": "20210628"}
 207         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 208
 209         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 210         runner.api.collections().get().execute.return_value = {
 211             "portable_data_hash": "99999999999999999999999999999993+99"}
 212
 213         tool = cmap({
 214             "inputs": [],
 215             "outputs": [],
 216             "hints": [{
 217                 "class": "ResourceRequirement",
 218                 "coresMin": 3,
 219                 "ramMin": 3000,
 220                 "tmpdirMin": 4000,
 221                 "outdirMin": 5000
 222             }, {
 223                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 224                 "keep_cache": 512
 225             }, {
 226                 "class": "http://arvados.org/cwl#APIRequirement",
 227             }, {
 228                 "class": "http://arvados.org/cwl#PartitionRequirement",
 229                 "partition": "blurb"
 230             }, {
 231                 "class": "http://arvados.org/cwl#IntermediateOutput",
 232                 "outputTTL": 7200
 233             }, {
 234                 "class": "WorkReuse",
 235                 "enableReuse": False
 236             }],
 237             "baseCommand": "ls",
 238             "id": "",
 239             "class": "CommandLineTool",
 240             "cwlVersion": "v1.2"
 241         })
 242
 243         loadingContext, runtimeContext = self.helper(runner)
 244         runtimeContext.name = "test_resource_requirements"
 245
 246         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 247         arvtool.formatgraph = None
 248         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 249             j.run(runtimeContext)
 250
 251         call_args, call_kwargs = runner.api.container_requests().create.call_args
 252
 253         call_body_expected = {
 254             'environment': {
 255                 'HOME': '/var/spool/cwl',
 256                 'TMPDIR': '/tmp'
 257             },
 258             'name': 'test_resource_requirements',
 259             'runtime_constraints': {
 260                 'vcpus': 3,
 261                 'ram': 3145728000,
 262                 'keep_cache_ram': 536870912,
 263                 'API': True
 264             },
 265             'use_existing': False,
 266             'priority': 500,
 267             'mounts': {
 268                 '/tmp': {'kind': 'tmp',
 269                          "capacity": 4194304000 },
 270                 '/var/spool/cwl': {'kind': 'tmp',
 271                                    "capacity": 5242880000 }
 272             },
 273             'state': 'Committed',
 274             'output_name': 'Output from step test_resource_requirements',
 275             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 276             'output_path': '/var/spool/cwl',
 277             'output_ttl': 7200,
 278             'container_image': '99999999999999999999999999999993+99',
 279             'command': ['ls'],
 280             'cwd': '/var/spool/cwl',
 281             'scheduling_parameters': {
 282                 'partitions': ['blurb']
 283             },
 284             'properties': {'cwl_input': {}},
 285             'secret_mounts': {},
 286             'output_storage_classes': ["default"]
 287         }
 288
 289         call_body = call_kwargs.get('body', None)
 290         self.assertNotEqual(None, call_body)
 291         for key in call_body:
 292             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 293
 294
 295     # The test passes some fields in builder.resources
 296     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 297     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 298     @mock.patch("arvados.collection.Collection")
 299     def test_initial_work_dir(self, collection_mock, keepdocker):
 300         runner = mock.MagicMock()
 301         runner.ignore_docker_for_reuse = False
 302         runner.intermediate_output_ttl = 0
 303         runner.secret_store = cwltool.secrets.SecretStore()
 304         runner.api._rootDesc = {"revision": "20210628"}
 305         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 306
 307         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 308         runner.api.collections().get().execute.return_value = {
 309             "portable_data_hash": "99999999999999999999999999999993+99"}
 310
 311         sourcemock = mock.MagicMock()
 312         def get_collection_mock(p):
 313             if "/" in p:
 314                 return (sourcemock, p.split("/", 1)[1])
 315             else:
 316                 return (sourcemock, "")
 317         runner.fs_access.get_collection.side_effect = get_collection_mock
 318
 319         vwdmock = mock.MagicMock()
 320         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 321
 322         tool = cmap({
 323             "inputs": [],
 324             "outputs": [],
 325             "hints": [{
 326                 "class": "InitialWorkDirRequirement",
 327                 "listing": [{
 328                     "class": "File",
 329                     "basename": "foo",
 330                     "location": "keep:99999999999999999999999999999995+99/bar"
 331                 },
 332                 {
 333                     "class": "Directory",
 334                     "basename": "foo2",
 335                     "location": "keep:99999999999999999999999999999995+99"
 336                 },
 337                 {
 338                     "class": "File",
 339                     "basename": "filename",
 340                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 341                 },
 342                 {
 343                     "class": "Directory",
 344                     "basename": "subdir",
 345                     "location": "keep:99999999999999999999999999999995+99/subdir"
 346                 }                        ]
 347             }],
 348             "baseCommand": "ls",
 349             "class": "CommandLineTool",
 350             "cwlVersion": "v1.2",
 351             "id": ""
 352         })
 353
 354         loadingContext, runtimeContext = self.helper(runner)
 355         runtimeContext.name = "test_initial_work_dir"
 356
 357         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 358
 359         arvtool.formatgraph = None
 360         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 361             j.run(runtimeContext)
 362
 363         call_args, call_kwargs = runner.api.container_requests().create.call_args
 364
 365         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 367         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 368         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 369
 370         call_body_expected = {
 371             'environment': {
 372                 'HOME': '/var/spool/cwl',
 373                 'TMPDIR': '/tmp'
 374             },
 375             'name': 'test_initial_work_dir',
 376             'runtime_constraints': {
 377                 'vcpus': 1,
 378                 'ram': 268435456
 379             },
 380             'use_existing': True,
 381             'priority': 500,
 382             'mounts': {
 383                 '/tmp': {'kind': 'tmp',
 384                          "capacity": 1073741824 },
 385                 '/var/spool/cwl': {'kind': 'tmp',
 386                                    "capacity": 1073741824 },
 387                 '/var/spool/cwl/foo': {
 388                     'kind': 'collection',
 389                     'path': 'foo',
 390                     'portable_data_hash': '99999999999999999999999999999996+99'
 391                 },
 392                 '/var/spool/cwl/foo2': {
 393                     'kind': 'collection',
 394                     'path': 'foo2',
 395                     'portable_data_hash': '99999999999999999999999999999996+99'
 396                 },
 397                 '/var/spool/cwl/filename': {
 398                     'kind': 'collection',
 399                     'path': 'filename',
 400                     'portable_data_hash': '99999999999999999999999999999996+99'
 401                 },
 402                 '/var/spool/cwl/subdir': {
 403                     'kind': 'collection',
 404                     'path': 'subdir',
 405                     'portable_data_hash': '99999999999999999999999999999996+99'
 406                 }
 407             },
 408             'state': 'Committed',
 409             'output_name': 'Output from step test_initial_work_dir',
 410             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 411             'output_path': '/var/spool/cwl',
 412             'output_ttl': 0,
 413             'container_image': '99999999999999999999999999999993+99',
 414             'command': ['ls'],
 415             'cwd': '/var/spool/cwl',
 416             'scheduling_parameters': {
 417             },
 418             'properties': {'cwl_input': {}},
 419             'secret_mounts': {},
 420             'output_storage_classes': ["default"]
 421         }
 422
 423         call_body = call_kwargs.get('body', None)
 424         self.assertNotEqual(None, call_body)
 425         for key in call_body:
 426             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 427
 428
 429     # Test redirecting stdin/stdout/stderr
 430     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 431     def test_redirects(self, keepdocker):
 432         runner = mock.MagicMock()
 433         runner.ignore_docker_for_reuse = False
 434         runner.intermediate_output_ttl = 0
 435         runner.secret_store = cwltool.secrets.SecretStore()
 436         runner.api._rootDesc = {"revision": "20210628"}
 437         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 438
 439         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 440         runner.api.collections().get().execute.return_value = {
 441             "portable_data_hash": "99999999999999999999999999999993+99"}
 442
 443         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 444
 445         tool = cmap({
 446             "inputs": [],
 447             "outputs": [],
 448             "baseCommand": "ls",
 449             "stdout": "stdout.txt",
 450             "stderr": "stderr.txt",
 451             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 452             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 453             "id": "",
 454             "class": "CommandLineTool",
 455             "cwlVersion": "v1.2"
 456         })
 457
 458         loadingContext, runtimeContext = self.helper(runner)
 459         runtimeContext.name = "test_run_redirect"
 460
 461         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 462         arvtool.formatgraph = None
 463         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 464             j.run(runtimeContext)
 465             runner.api.container_requests().create.assert_called_with(
 466                 body=JsonDiffMatcher({
 467                     'environment': {
 468                         'HOME': '/var/spool/cwl',
 469                         'TMPDIR': '/tmp'
 470                     },
 471                     'name': 'test_run_redirect',
 472                     'runtime_constraints': {
 473                         'vcpus': 1,
 474                         'ram': 268435456
 475                     },
 476                     'use_existing': True,
 477                     'priority': 500,
 478                     'mounts': {
 479                         '/tmp': {'kind': 'tmp',
 480                                  "capacity": 1073741824 },
 481                         '/var/spool/cwl': {'kind': 'tmp',
 482                                            "capacity": 1073741824 },
 483                         "stderr": {
 484                             "kind": "file",
 485                             "path": "/var/spool/cwl/stderr.txt"
 486                         },
 487                         "stdin": {
 488                             "kind": "collection",
 489                             "path": "file.txt",
 490                             "portable_data_hash": "99999999999999999999999999999996+99"
 491                         },
 492                         "stdout": {
 493                             "kind": "file",
 494                             "path": "/var/spool/cwl/stdout.txt"
 495                         },
 496                     },
 497                     'state': 'Committed',
 498                     "output_name": "Output from step test_run_redirect",
 499                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 500                     'output_path': '/var/spool/cwl',
 501                     'output_ttl': 0,
 502                     'container_image': '99999999999999999999999999999993+99',
 503                     'command': ['ls', '/var/spool/cwl'],
 504                     'cwd': '/var/spool/cwl',
 505                     'scheduling_parameters': {},
 506                     'properties': {'cwl_input': {}},
 507                     'secret_mounts': {},
 508                     'output_storage_classes': ["default"]
 509                 }))
 510
 511     @mock.patch("arvados.collection.Collection")
 512     def test_done(self, col):
 513         api = mock.MagicMock()
 514
 515         runner = mock.MagicMock()
 516         runner.api = api
 517         runner.num_retries = 0
 518         runner.ignore_docker_for_reuse = False
 519         runner.intermediate_output_ttl = 0
 520         runner.secret_store = cwltool.secrets.SecretStore()
 521
 522         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 523
 524         runner.api.containers().get().execute.return_value = {"state":"Complete",
 525                                                               "output": "abc+123",
 526                                                               "exit_code": 0}
 527
 528         # Need to noop-out the close method otherwise it gets
 529         # discarded when closed and we can't call getvalue() to check
 530         # it.
 531         class NoopCloseStringIO(io.StringIO):
 532             def close(self):
 533                 pass
 534
 535         usage_report = NoopCloseStringIO()
 536         def colreader_action(name, mode):
 537             nonlocal usage_report
 538             if name == "node.json":
 539                 return io.StringIO("""{
 540     "ProviderType": "c5.large",
 541     "VCPUs": 2,
 542     "RAM": 4294967296,
 543     "IncludedScratch": 8000000000000,
 544     "AddedScratch": 0,
 545     "Price": 0.085,
 546     "Preemptible": false,
 547     "CUDA": {
 548         "DriverVersion": "",
 549         "HardwareCapability": "",
 550         "DeviceCount": 0
 551     }
 552 }""")
 553             if name == 'crunchstat.txt':
 554                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 555             if name == 'arv-mount.txt':
 556                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 557             if name == 'usage_report.html':
 558                 return usage_report
 559             return None
 560
 561         col().open.side_effect = colreader_action
 562         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 563
 564         loadingContext, runtimeContext = self.helper(runner)
 565
 566         arvjob = arvados_cwl.ArvadosContainer(runner,
 567                                               runtimeContext,
 568                                               [],
 569                                               mock.MagicMock(),
 570                                               {},
 571                                               None,
 572                                               [],
 573                                               [],
 574                                               "testjob")
 575         arvjob.output_callback = mock.MagicMock()
 576         arvjob.collect_outputs = mock.MagicMock()
 577         arvjob.successCodes = [0]
 578         arvjob.outdir = "/var/spool/cwl"
 579         arvjob.output_ttl = 3600
 580         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 581
 582         arvjob.collect_outputs.return_value = {"out": "stuff"}
 583
 584         arvjob.done({
 585             "state": "Final",
 586             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 587             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 588             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 589             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 590             "modified_at": "2017-05-26T12:01:22Z",
 591             "properties": {},
 592             "name": "testjob"
 593         })
 594
 595         self.assertFalse(api.collections().create.called)
 596         self.assertFalse(runner.runtime_status_error.called)
 597
 598         # Assert that something was written to the usage report
 599         self.assertTrue(len(usage_report.getvalue()) > 0)
 600
 601         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 602         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 603         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 604
 605         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 606                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 607
 608
 609     # Test to make sure we dont call runtime_status_update if we already did
 610     # some where higher up in the call stack
 611     @mock.patch("arvados_cwl.util.get_current_container")
 612     def test_recursive_runtime_status_update(self, gcc_mock):
 613         self.setup_and_test_container_executor_and_logging(gcc_mock)
 614         root_logger = logging.getLogger('')
 615
 616         # get_current_container is invoked when we call runtime_status_update
 617         # so try and log again!
 618         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 619         try:
 620             root_logger.error("First Error")
 621         except RuntimeError:
 622             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 623
 624
 625     # Test to make sure that an exception raised from
 626     # get_current_container doesn't cause the logger to raise an
 627     # exception
 628     @mock.patch("arvados_cwl.util.get_current_container")
 629     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 630         self.setup_and_test_container_executor_and_logging(gcc_mock)
 631         root_logger = logging.getLogger('')
 632
 633         # get_current_container is invoked when we call
 634         # runtime_status_update, it is going to also raise an
 635         # exception.
 636         gcc_mock.side_effect = Exception("Second Error")
 637         try:
 638             root_logger.error("First Error")
 639         except Exception:
 640             self.fail("Exception in logger should not propagate")
 641         self.assertTrue(gcc_mock.called)
 642
 643     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 644     @mock.patch("arvados_cwl.util.get_current_container")
 645     @mock.patch("arvados.collection.CollectionReader")
 646     @mock.patch("arvados.collection.Collection")
 647     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 648         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 649
 650         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 651         self.assertTrue(gcc_mock.called)
 652
 653         runner.num_retries = 0
 654         runner.ignore_docker_for_reuse = False
 655         runner.intermediate_output_ttl = 0
 656         runner.secret_store = cwltool.secrets.SecretStore()
 657         runner.label = mock.MagicMock()
 658         runner.label.return_value = '[container testjob]'
 659
 660         runner.api.containers().get().execute.return_value = {
 661             "state":"Complete",
 662             "output": "abc+123",
 663             "exit_code": 1,
 664             "log": "def+234"
 665         }
 666
 667         col().open.return_value = []
 668
 669         loadingContext, runtimeContext = self.helper(runner)
 670
 671         arvjob = arvados_cwl.ArvadosContainer(runner,
 672                                               runtimeContext,
 673                                               [],
 674                                               mock.MagicMock(),
 675                                               {},
 676                                               None,
 677                                               [],
 678                                               [],
 679                                               "testjob")
 680         arvjob.output_callback = mock.MagicMock()
 681         arvjob.collect_outputs = mock.MagicMock()
 682         arvjob.successCodes = [0]
 683         arvjob.outdir = "/var/spool/cwl"
 684         arvjob.output_ttl = 3600
 685         arvjob.collect_outputs.return_value = {"out": "stuff"}
 686
 687         arvjob.done({
 688             "state": "Final",
 689             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 690             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 691             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 692             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 693             "modified_at": "2017-05-26T12:01:22Z",
 694             "properties": {}
 695         })
 696
 697         rts_mock.assert_has_calls([
 698             mock.call('error',
 699                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 700                       '  ** log is empty **'
 701                       ),
 702             mock.call('warning',
 703                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 704         )])
 705         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 706
 707     # The test passes no builder.resources
 708     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 709     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 710     def test_mounts(self, keepdocker):
 711         runner = mock.MagicMock()
 712         runner.ignore_docker_for_reuse = False
 713         runner.intermediate_output_ttl = 0
 714         runner.secret_store = cwltool.secrets.SecretStore()
 715         runner.api._rootDesc = {"revision": "20210628"}
 716         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 717
 718         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 719         runner.api.collections().get().execute.return_value = {
 720             "portable_data_hash": "99999999999999999999999999999994+99",
 721             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 722
 723         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 724
 725         tool = cmap({
 726             "inputs": [
 727                 {"id": "p1",
 728                  "type": "Directory"}
 729             ],
 730             "outputs": [],
 731             "baseCommand": "ls",
 732             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 733             "id": "",
 734             "class": "CommandLineTool",
 735             "cwlVersion": "v1.2"
 736         })
 737
 738         loadingContext, runtimeContext = self.helper(runner)
 739         runtimeContext.name = "test_run_mounts"
 740
 741         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 742         arvtool.formatgraph = None
 743         job_order = {
 744             "p1": {
 745                 "class": "Directory",
 746                 "location": "keep:99999999999999999999999999999994+44",
 747                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 748                 "listing": [
 749                     {
 750                         "class": "File",
 751                         "location": "keep:99999999999999999999999999999994+44/file1",
 752                     },
 753                     {
 754                         "class": "File",
 755                         "location": "keep:99999999999999999999999999999994+44/file2",
 756                     }
 757                 ]
 758             }
 759         }
 760         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 761             j.run(runtimeContext)
 762             runner.api.container_requests().create.assert_called_with(
 763                 body=JsonDiffMatcher({
 764                     'environment': {
 765                         'HOME': '/var/spool/cwl',
 766                         'TMPDIR': '/tmp'
 767                     },
 768                     'name': 'test_run_mounts',
 769                     'runtime_constraints': {
 770                         'vcpus': 1,
 771                         'ram': 268435456
 772                     },
 773                     'use_existing': True,
 774                     'priority': 500,
 775                     'mounts': {
 776                         "/keep/99999999999999999999999999999994+44": {
 777                             "kind": "collection",
 778                             "portable_data_hash": "99999999999999999999999999999994+44",
 779                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 780                         },
 781                         '/tmp': {'kind': 'tmp',
 782                                  "capacity": 1073741824 },
 783                         '/var/spool/cwl': {'kind': 'tmp',
 784                                            "capacity": 1073741824 }
 785                     },
 786                     'state': 'Committed',
 787                     'output_name': 'Output from step test_run_mounts',
 788                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 789                     'output_path': '/var/spool/cwl',
 790                     'output_ttl': 0,
 791                     'container_image': '99999999999999999999999999999994+99',
 792                     'command': ['ls', '/var/spool/cwl'],
 793                     'cwd': '/var/spool/cwl',
 794                     'scheduling_parameters': {},
 795                     'properties': {'cwl_input': {
 796                         "p1": {
 797                             "basename": "99999999999999999999999999999994+44",
 798                             "class": "Directory",
 799                             "dirname": "/keep",
 800                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 801                             "listing": [
 802                                 {
 803                                     "basename": "file1",
 804                                     "class": "File",
 805                                     "dirname": "/keep/99999999999999999999999999999994+44",
 806                                     "location": "keep:99999999999999999999999999999994+44/file1",
 807                                     "nameext": "",
 808                                     "nameroot": "file1",
 809                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 810                                     "size": 0
 811                                 },
 812                                 {
 813                                     "basename": "file2",
 814                                     "class": "File",
 815                                     "dirname": "/keep/99999999999999999999999999999994+44",
 816                                     "location": "keep:99999999999999999999999999999994+44/file2",
 817                                     "nameext": "",
 818                                     "nameroot": "file2",
 819                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 820                                     "size": 0
 821                                 }
 822                             ],
 823                             "location": "keep:99999999999999999999999999999994+44",
 824                             "path": "/keep/99999999999999999999999999999994+44"
 825                         }
 826                     }},
 827                     'secret_mounts': {},
 828                     'output_storage_classes': ["default"]
 829                 }))
 830
 831     # The test passes no builder.resources
 832     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 833     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 834     def test_secrets(self, keepdocker):
 835         arvados_cwl.add_arv_hints()
 836         runner = mock.MagicMock()
 837         runner.ignore_docker_for_reuse = False
 838         runner.intermediate_output_ttl = 0
 839         runner.secret_store = cwltool.secrets.SecretStore()
 840         runner.api._rootDesc = {"revision": "20210628"}
 841         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 842
 843         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 844         runner.api.collections().get().execute.return_value = {
 845             "portable_data_hash": "99999999999999999999999999999993+99"}
 846
 847         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 848
 849         tool = cmap({"arguments": ["md5sum", "example.conf"],
 850                      "class": "CommandLineTool",
 851                      "cwlVersion": "v1.2",
 852                      "hints": [
 853                          {
 854                              "class": "http://commonwl.org/cwltool#Secrets",
 855                              "secrets": [
 856                                  "#secret_job.cwl/pw"
 857                              ]
 858                          }
 859                      ],
 860                      "id": "",
 861                      "inputs": [
 862                          {
 863                              "id": "#secret_job.cwl/pw",
 864                              "type": "string"
 865                          }
 866                      ],
 867                      "outputs": [
 868                      ],
 869                      "requirements": [
 870                          {
 871                              "class": "InitialWorkDirRequirement",
 872                              "listing": [
 873                                  {
 874                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 875                                      "entryname": "example.conf"
 876                                  }
 877                              ]
 878                          }
 879                      ]})
 880
 881         loadingContext, runtimeContext = self.helper(runner)
 882         runtimeContext.name = "test_secrets"
 883
 884         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 885         arvtool.formatgraph = None
 886
 887         job_order = {"pw": "blorp"}
 888         runner.secret_store.store(["pw"], job_order)
 889
 890         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 891             j.run(runtimeContext)
 892             runner.api.container_requests().create.assert_called_with(
 893                 body=JsonDiffMatcher({
 894                     'environment': {
 895                         'HOME': '/var/spool/cwl',
 896                         'TMPDIR': '/tmp'
 897                     },
 898                     'name': 'test_secrets',
 899                     'runtime_constraints': {
 900                         'vcpus': 1,
 901                         'ram': 268435456
 902                     },
 903                     'use_existing': True,
 904                     'priority': 500,
 905                     'mounts': {
 906                         '/tmp': {'kind': 'tmp',
 907                                  "capacity": 1073741824
 908                              },
 909                         '/var/spool/cwl': {'kind': 'tmp',
 910                                            "capacity": 1073741824 }
 911                     },
 912                     'state': 'Committed',
 913                     'output_name': 'Output from step test_secrets',
 914                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 915                     'output_path': '/var/spool/cwl',
 916                     'output_ttl': 0,
 917                     'container_image': '99999999999999999999999999999993+99',
 918                     'command': ['md5sum', 'example.conf'],
 919                     'cwd': '/var/spool/cwl',
 920                     'scheduling_parameters': {},
 921                     'properties': {'cwl_input': job_order},
 922                     "secret_mounts": {
 923                         "/var/spool/cwl/example.conf": {
 924                             "content": "username: user\npassword: blorp\n",
 925                             "kind": "text"
 926                         }
 927                     },
 928                     'output_storage_classes': ["default"]
 929                 }))
 930
 931     # The test passes no builder.resources
 932     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 933     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 934     def test_timelimit(self, keepdocker):
 935         runner = mock.MagicMock()
 936         runner.ignore_docker_for_reuse = False
 937         runner.intermediate_output_ttl = 0
 938         runner.secret_store = cwltool.secrets.SecretStore()
 939         runner.api._rootDesc = {"revision": "20210628"}
 940         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 941
 942         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 943         runner.api.collections().get().execute.return_value = {
 944             "portable_data_hash": "99999999999999999999999999999993+99"}
 945
 946         tool = cmap({
 947             "inputs": [],
 948             "outputs": [],
 949             "baseCommand": "ls",
 950             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 951             "id": "",
 952             "cwlVersion": "v1.2",
 953             "class": "CommandLineTool",
 954             "hints": [
 955                 {
 956                     "class": "ToolTimeLimit",
 957                     "timelimit": 42
 958                 }
 959             ]
 960         })
 961
 962         loadingContext, runtimeContext = self.helper(runner)
 963         runtimeContext.name = "test_timelimit"
 964
 965         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 966         arvtool.formatgraph = None
 967
 968         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 969             j.run(runtimeContext)
 970
 971         _, kwargs = runner.api.container_requests().create.call_args
 972         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 973
 974
 975     # The test passes no builder.resources
 976     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 977     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 978     def test_setting_storage_class(self, keepdocker):
 979         #arv_docker_clear_cache()
 980
 981         runner = mock.MagicMock()
 982         runner.ignore_docker_for_reuse = False
 983         runner.intermediate_output_ttl = 0
 984         runner.secret_store = cwltool.secrets.SecretStore()
 985         runner.api._rootDesc = {"revision": "20210628"}
 986         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 987
 988         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 989         runner.api.collections().get().execute.return_value = {
 990             "portable_data_hash": "99999999999999999999999999999993+99"}
 991
 992         tool = cmap({
 993             "inputs": [],
 994             "outputs": [],
 995             "baseCommand": "ls",
 996             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 997             "id": "",
 998             "cwlVersion": "v1.2",
 999             "class": "CommandLineTool",
1000             "hints": [
1001                 {
1002                     "class": "http://arvados.org/cwl#OutputStorageClass",
1003                     "finalStorageClass": ["baz_sc", "qux_sc"],
1004                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1005                 }
1006             ]
1007         })
1008
1009         loadingContext, runtimeContext = self.helper(runner, True)
1010
1011         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1012         arvtool.formatgraph = None
1013
1014         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1015             j.run(runtimeContext)
1016             runner.api.container_requests().create.assert_called_with(
1017                 body=JsonDiffMatcher({
1018                     'environment': {
1019                         'HOME': '/var/spool/cwl',
1020                         'TMPDIR': '/tmp'
1021                     },
1022                     'name': 'test_run_True',
1023                     'runtime_constraints': {
1024                         'vcpus': 1,
1025                         'ram': 268435456
1026                     },
1027                     'use_existing': True,
1028                     'priority': 500,
1029                     'mounts': {
1030                         '/tmp': {'kind': 'tmp',
1031                                  "capacity": 1073741824
1032                              },
1033                         '/var/spool/cwl': {'kind': 'tmp',
1034                                            "capacity": 1073741824 }
1035                     },
1036                     'state': 'Committed',
1037                     'output_name': 'Output from step test_run_True',
1038                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1039                     'output_path': '/var/spool/cwl',
1040                     'output_ttl': 0,
1041                     'container_image': '99999999999999999999999999999993+99',
1042                     'command': ['ls', '/var/spool/cwl'],
1043                     'cwd': '/var/spool/cwl',
1044                     'scheduling_parameters': {},
1045                     'properties': {'cwl_input': {}},
1046                     'secret_mounts': {},
1047                     'output_storage_classes': ["foo_sc", "bar_sc"]
1048                 }))
1049
1050
1051     # The test passes no builder.resources
1052     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1053     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1054     def test_setting_process_properties(self, keepdocker):
1055         #arv_docker_clear_cache()
1056
1057         runner = mock.MagicMock()
1058         runner.ignore_docker_for_reuse = False
1059         runner.intermediate_output_ttl = 0
1060         runner.secret_store = cwltool.secrets.SecretStore()
1061         runner.api._rootDesc = {"revision": "20210628"}
1062         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1063
1064         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1065         runner.api.collections().get().execute.return_value = {
1066             "portable_data_hash": "99999999999999999999999999999993+99"}
1067
1068         tool = cmap({
1069             "inputs": [
1070                 {"id": "x", "type": "string"}],
1071             "outputs": [],
1072             "baseCommand": "ls",
1073             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1074             "id": "",
1075             "class": "CommandLineTool",
1076             "cwlVersion": "v1.2",
1077             "hints": [
1078             {
1079                 "class": "http://arvados.org/cwl#ProcessProperties",
1080                 "processProperties": [
1081                     {"propertyName": "foo",
1082                      "propertyValue": "bar"},
1083                     {"propertyName": "baz",
1084                      "propertyValue": "$(inputs.x)"},
1085                     {"propertyName": "quux",
1086                      "propertyValue": {
1087                          "q1": 1,
1088                          "q2": 2
1089                      }
1090                     }
1091                 ],
1092             }
1093         ]
1094         })
1095
1096         loadingContext, runtimeContext = self.helper(runner, True)
1097
1098         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1099         arvtool.formatgraph = None
1100
1101         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1102             j.run(runtimeContext)
1103             runner.api.container_requests().create.assert_called_with(
1104                 body=JsonDiffMatcher({
1105                     'environment': {
1106                         'HOME': '/var/spool/cwl',
1107                         'TMPDIR': '/tmp'
1108                     },
1109                     'name': 'test_run_True',
1110                     'runtime_constraints': {
1111                         'vcpus': 1,
1112                         'ram': 268435456
1113                     },
1114                     'use_existing': True,
1115                     'priority': 500,
1116                     'mounts': {
1117                         '/tmp': {'kind': 'tmp',
1118                                  "capacity": 1073741824
1119                              },
1120                         '/var/spool/cwl': {'kind': 'tmp',
1121                                            "capacity": 1073741824 }
1122                     },
1123                     'state': 'Committed',
1124                     'output_name': 'Output from step test_run_True',
1125                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1126                     'output_path': '/var/spool/cwl',
1127                     'output_ttl': 0,
1128                     'container_image': '99999999999999999999999999999993+99',
1129                     'command': ['ls', '/var/spool/cwl'],
1130                     'cwd': '/var/spool/cwl',
1131                     'scheduling_parameters': {},
1132                     'properties': {
1133                         "baz": "blorp",
1134                         "cwl_input": {"x": "blorp"},
1135                         "foo": "bar",
1136                         "quux": {
1137                             "q1": 1,
1138                             "q2": 2
1139                         }
1140                     },
1141                     'secret_mounts': {},
1142                     'output_storage_classes': ["default"]
1143                 }))
1144
1145
1146     # The test passes no builder.resources
1147     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1148     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1149     def test_cuda_requirement(self, keepdocker):
1150         arvados_cwl.add_arv_hints()
1151         #arv_docker_clear_cache()
1152
1153         runner = mock.MagicMock()
1154         runner.ignore_docker_for_reuse = False
1155         runner.intermediate_output_ttl = 0
1156         runner.secret_store = cwltool.secrets.SecretStore()
1157         runner.api._rootDesc = {"revision": "20210628"}
1158         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1159
1160         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1161         runner.api.collections().get().execute.return_value = {
1162             "portable_data_hash": "99999999999999999999999999999993+99"}
1163
1164         test_cwl_req = [{
1165                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1166                 "cudaVersionMin": "11.0",
1167                 "cudaComputeCapability": "9.0",
1168             }, {
1169                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1170                 "cudaVersionMin": "11.0",
1171                 "cudaComputeCapability": "9.0",
1172                 "cudaDeviceCountMin": 2
1173             }, {
1174                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1175                 "cudaVersionMin": "11.0",
1176                 "cudaComputeCapability": ["4.0", "5.0"],
1177                 "cudaDeviceCountMin": 2
1178             }]
1179
1180         test_arv_req = [{
1181             'device_count': 1,
1182             'driver_version': "11.0",
1183             'hardware_capability': "9.0"
1184         }, {
1185             'device_count': 2,
1186             'driver_version': "11.0",
1187             'hardware_capability': "9.0"
1188         }, {
1189             'device_count': 2,
1190             'driver_version': "11.0",
1191             'hardware_capability': "4.0"
1192         }]
1193
1194         for test_case in range(0, len(test_cwl_req)):
1195
1196             tool = cmap({
1197                 "inputs": [],
1198                 "outputs": [],
1199                 "baseCommand": "nvidia-smi",
1200                 "arguments": [],
1201                 "id": "",
1202                 "cwlVersion": "v1.2",
1203                 "class": "CommandLineTool",
1204                 "requirements": [test_cwl_req[test_case]]
1205             })
1206
1207             loadingContext, runtimeContext = self.helper(runner, True)
1208
1209             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1210             arvtool.formatgraph = None
1211
1212             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1213                 j.run(runtimeContext)
1214                 runner.api.container_requests().create.assert_called_with(
1215                     body=JsonDiffMatcher({
1216                         'environment': {
1217                             'HOME': '/var/spool/cwl',
1218                             'TMPDIR': '/tmp'
1219                         },
1220                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1221                         'runtime_constraints': {
1222                             'vcpus': 1,
1223                             'ram': 268435456,
1224                             'cuda': test_arv_req[test_case]
1225                         },
1226                         'use_existing': True,
1227                         'priority': 500,
1228                         'mounts': {
1229                             '/tmp': {'kind': 'tmp',
1230                                      "capacity": 1073741824
1231                                  },
1232                             '/var/spool/cwl': {'kind': 'tmp',
1233                                                "capacity": 1073741824 }
1234                         },
1235                         'state': 'Committed',
1236                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1237                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1238                         'output_path': '/var/spool/cwl',
1239                         'output_ttl': 0,
1240                         'container_image': '99999999999999999999999999999993+99',
1241                         'command': ['nvidia-smi'],
1242                         'cwd': '/var/spool/cwl',
1243                         'scheduling_parameters': {},
1244                         'properties': {'cwl_input': {}},
1245                         'secret_mounts': {},
1246                         'output_storage_classes': ["default"]
1247                     }))
1248
1249
1250     # The test passes no builder.resources
1251     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1252     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1253     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1254     def test_match_local_docker(self, keepdocker, determine_image_id):
1255         arvados_cwl.add_arv_hints()
1256
1257         runner = mock.MagicMock()
1258         runner.ignore_docker_for_reuse = False
1259         runner.intermediate_output_ttl = 0
1260         runner.secret_store = cwltool.secrets.SecretStore()
1261         runner.api._rootDesc = {"revision": "20210628"}
1262         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1263
1264         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1265                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1266         determine_image_id.side_effect = lambda x: "123"
1267         def execute(uuid):
1268             ex = mock.MagicMock()
1269             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1270                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1271             ex.execute.return_value = lookup[uuid]
1272             return ex
1273         runner.api.collections().get.side_effect = execute
1274
1275         tool = cmap({
1276             "inputs": [],
1277             "outputs": [],
1278             "baseCommand": "echo",
1279             "arguments": [],
1280             "id": "",
1281             "cwlVersion": "v1.0",
1282             "class": "org.w3id.cwl.cwl.CommandLineTool"
1283         })
1284
1285         loadingContext, runtimeContext = self.helper(runner, True)
1286
1287         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1288         arvtool.formatgraph = None
1289
1290         container_request = {
1291             'environment': {
1292                 'HOME': '/var/spool/cwl',
1293                 'TMPDIR': '/tmp'
1294             },
1295             'name': 'test_run_True',
1296             'runtime_constraints': {
1297                 'vcpus': 1,
1298                 'ram': 1073741824,
1299             },
1300             'use_existing': True,
1301             'priority': 500,
1302             'mounts': {
1303                 '/tmp': {'kind': 'tmp',
1304                          "capacity": 1073741824
1305                          },
1306                 '/var/spool/cwl': {'kind': 'tmp',
1307                                    "capacity": 1073741824 }
1308             },
1309             'state': 'Committed',
1310             'output_name': 'Output from step test_run_True',
1311             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1312             'output_path': '/var/spool/cwl',
1313             'output_ttl': 0,
1314             'container_image': '99999999999999999999999999999994+99',
1315             'command': ['echo'],
1316             'cwd': '/var/spool/cwl',
1317             'scheduling_parameters': {},
1318             'properties': {'cwl_input': {}},
1319             'secret_mounts': {},
1320             'output_storage_classes': ["default"]
1321         }
1322
1323         runtimeContext.match_local_docker = False
1324         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1325             j.run(runtimeContext)
1326             runner.api.container_requests().create.assert_called_with(
1327                 body=JsonDiffMatcher(container_request))
1328
1329         runtimeContext.cached_docker_lookups.clear()
1330         runtimeContext.match_local_docker = True
1331         container_request['container_image'] = '99999999999999999999999999999993+99'
1332         container_request['name'] = 'test_run_True_2'
1333         container_request['output_name'] = 'Output from step test_run_True_2'
1334         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1335             j.run(runtimeContext)
1336             runner.api.container_requests().create.assert_called_with(
1337                 body=JsonDiffMatcher(container_request))
1338
1339
1340     # The test passes no builder.resources
1341     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1342     @parameterized.expand([
1343         ("None, None",   None,  None,  None),
1344         ("None, True",   None,  True,  True),
1345         ("None, False",  None,  False, False),
1346         ("False, None",  False, None,  False),
1347         ("False, True",  False, True,  False),
1348         ("False, False", False, False, False),
1349         ("True, None",   True,  None,  True),
1350         ("True, True",   True,  True,  True),
1351         ("True, False",  True,  False, False),
1352     ])
1353     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1354     def test_run_preemptible_hint(self, _, enable_preemptible, preemptible_hint,
1355                                   preemptible_setting, keepdocker):
1356         arvados_cwl.add_arv_hints()
1357
1358         runner = mock.MagicMock()
1359         runner.ignore_docker_for_reuse = False
1360         runner.intermediate_output_ttl = 0
1361         runner.secret_store = cwltool.secrets.SecretStore()
1362         runner.api._rootDesc = {"revision": "20210628"}
1363         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1364
1365         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1366         runner.api.collections().get().execute.return_value = {
1367             "portable_data_hash": "99999999999999999999999999999993+99"}
1368
1369         if preemptible_hint is not None:
1370             hints = [{
1371                 "class": "http://arvados.org/cwl#UsePreemptible",
1372                 "usePreemptible": preemptible_hint
1373             }]
1374         else:
1375             hints = []
1376
1377         tool = cmap({
1378             "inputs": [],
1379             "outputs": [],
1380             "baseCommand": "ls",
1381             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1382             "id": "",
1383             "class": "CommandLineTool",
1384             "cwlVersion": "v1.2",
1385             "hints": hints
1386         })
1387
1388         loadingContext, runtimeContext = self.helper(runner)
1389
1390         runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1391         runtimeContext.enable_preemptible = enable_preemptible
1392
1393         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1394         arvtool.formatgraph = None
1395
1396         # Test the interactions between --enable/disable-preemptible
1397         # and UsePreemptible hint
1398
1399         sched = {}
1400         if preemptible_setting is not None:
1401             sched['preemptible'] = preemptible_setting
1402
1403         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1404             j.run(runtimeContext)
1405             runner.api.container_requests().create.assert_called_with(
1406                 body=JsonDiffMatcher({
1407                     'environment': {
1408                         'HOME': '/var/spool/cwl',
1409                         'TMPDIR': '/tmp'
1410                     },
1411                     'name': runtimeContext.name,
1412                     'runtime_constraints': {
1413                         'vcpus': 1,
1414                         'ram': 268435456
1415                     },
1416                     'use_existing': True,
1417                     'priority': 500,
1418                     'mounts': {
1419                         '/tmp': {'kind': 'tmp',
1420                                  "capacity": 1073741824
1421                              },
1422                         '/var/spool/cwl': {'kind': 'tmp',
1423                                            "capacity": 1073741824 }
1424                     },
1425                     'state': 'Committed',
1426                     'output_name': 'Output from step '+runtimeContext.name,
1427                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1428                     'output_path': '/var/spool/cwl',
1429                     'output_ttl': 0,
1430                     'container_image': '99999999999999999999999999999993+99',
1431                     'command': ['ls', '/var/spool/cwl'],
1432                     'cwd': '/var/spool/cwl',
1433                     'scheduling_parameters': sched,
1434                     'properties': {'cwl_input': {}},
1435                     'secret_mounts': {},
1436                     'output_storage_classes': ["default"]
1437                 }))
1438
1439
1440     # The test passes no builder.resources
1441     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1442     @parameterized.expand([
1443         ("None, None", None, None, False),
1444         ("None, True", None, True, True),
1445         ("None, False", None, False, False),
1446         ("False, None", False, None, False),
1447         ("False, True", False, True, False),  # command line overrides hint
1448         ("False, False", False, False, False),
1449         ("True, None", True, None, True),
1450         ("True, True", True, True, True),
1451         ("True, False", True, False, False),  # hint overrides command line
1452     ])
1453     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1454     def test_spot_retry(self, _, enable_resubmit_non_preemptible,
1455                         preemption_behavior_hint,
1456                         expect_resubmit_behavior,
1457                         keepdocker):
1458         arvados_cwl.add_arv_hints()
1459         #arv_docker_clear_cache()
1460
1461         runner = mock.MagicMock()
1462         runner.ignore_docker_for_reuse = False
1463         runner.intermediate_output_ttl = 0
1464         runner.secret_store = cwltool.secrets.SecretStore()
1465         runner.api._rootDesc = {"revision": "20210628"}
1466         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1467
1468         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1469         runner.api.collections().get().execute.return_value = {
1470             "portable_data_hash": "99999999999999999999999999999993+99"}
1471
1472
1473         hints = [{
1474             "class": "http://arvados.org/cwl#UsePreemptible",
1475             "usePreemptible": True
1476         }]
1477
1478         if preemption_behavior_hint is not None:
1479             hints.append({
1480                 "class": "http://arvados.org/cwl#PreemptionBehavior",
1481                 "resubmitNonPreemptible": preemption_behavior_hint
1482             })
1483
1484         tool = cmap({
1485             "inputs": [],
1486             "outputs": [],
1487             "baseCommand": "ls",
1488             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1489             "id": "",
1490             "class": "CommandLineTool",
1491             "cwlVersion": "v1.2",
1492             "hints": hints
1493         })
1494
1495         loadingContext, runtimeContext = self.helper(runner)
1496
1497         runtimeContext.name = 'test_spot_retry_'+str(enable_resubmit_non_preemptible)+str(preemption_behavior_hint)
1498         runtimeContext.enable_resubmit_non_preemptible = enable_resubmit_non_preemptible
1499
1500         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1501         arvtool.formatgraph = None
1502
1503         # Test the interactions between --enable/disable-preemptible
1504         # and UsePreemptible hint
1505
1506         expect_container_request = {
1507                     'environment': {
1508                         'HOME': '/var/spool/cwl',
1509                         'TMPDIR': '/tmp'
1510                     },
1511                     'name': runtimeContext.name,
1512                     'runtime_constraints': {
1513                         'vcpus': 1,
1514                         'ram': 268435456
1515                     },
1516                     'use_existing': True,
1517                     'priority': 500,
1518                     'mounts': {
1519                         '/tmp': {'kind': 'tmp',
1520                                  "capacity": 1073741824
1521                              },
1522                         '/var/spool/cwl': {'kind': 'tmp',
1523                                            "capacity": 1073741824 }
1524                     },
1525                     'state': 'Committed',
1526                     'output_name': 'Output from step '+runtimeContext.name,
1527                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1528                     'output_path': '/var/spool/cwl',
1529                     'output_ttl': 0,
1530                     'container_image': '99999999999999999999999999999993+99',
1531                     'command': ['ls', '/var/spool/cwl'],
1532                     'cwd': '/var/spool/cwl',
1533                     'scheduling_parameters': {'preemptible': True},
1534                     'properties': {'cwl_input': {}},
1535                     'secret_mounts': {},
1536                     'output_storage_classes': ["default"],
1537                 }
1538
1539         expect_resubmit_container_request = expect_container_request.copy()
1540         expect_resubmit_container_request['scheduling_parameters'] = {'preemptible': False}
1541
1542         if expect_resubmit_behavior:
1543             expect_container_request['container_count_max'] = 1
1544
1545         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1546             j.run(runtimeContext)
1547             runner.api.container_requests().create.assert_called_with(
1548                 body=JsonDiffMatcher(expect_container_request))
1549         runner.api.containers().get().execute.return_value = {
1550             "state":"Cancelled",
1551             "output": "abc+123",
1552             "exit_code": 1,
1553             "log": "def+234",
1554             "runtime_status": {
1555                 "preemptionNotice": "bye bye"
1556             }
1557         }
1558         j.done({
1559             "state": "Final",
1560             "log_uuid": "",
1561             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
1562             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
1563             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
1564             "modified_at": "2017-05-26T12:01:22Z",
1565             "properties": {},
1566             "name": "testjob"
1567         })
1568         if expect_resubmit_behavior:
1569             runner.api.container_requests().create.assert_called_with(
1570                 body=JsonDiffMatcher(expect_resubmit_container_request))
1571
1572     @parameterized.expand([
1573         ("20210628",),
1574         ("20220510",),
1575     ])
1576     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1577     def test_output_properties(self, rev, keepdocker):
1578         arvados_cwl.add_arv_hints()
1579         runner = mock.MagicMock()
1580         runner.ignore_docker_for_reuse = False
1581         runner.intermediate_output_ttl = 0
1582         runner.secret_store = cwltool.secrets.SecretStore()
1583         runner.api._rootDesc = {"revision": rev}
1584         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1585
1586         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1587         runner.api.collections().get().execute.return_value = {
1588             "portable_data_hash": "99999999999999999999999999999993+99"}
1589
1590         tool = cmap({
1591             "inputs": [{
1592                 "id": "inp",
1593                 "type": "string"
1594             }],
1595             "outputs": [],
1596             "baseCommand": "ls",
1597             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1598             "id": "",
1599             "cwlVersion": "v1.2",
1600             "class": "CommandLineTool",
1601             "hints": [
1602                 {
1603                     "class": "http://arvados.org/cwl#OutputCollectionProperties",
1604                     "outputProperties": {
1605                         "foo": "bar",
1606                         "baz": "$(inputs.inp)"
1607                     }
1608                 }
1609             ]
1610         })
1611
1612         loadingContext, runtimeContext = self.helper(runner)
1613         runtimeContext.name = "test_timelimit"
1614
1615         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1616         arvtool.formatgraph = None
1617
1618         for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1619             j.run(runtimeContext)
1620
1621         _, kwargs = runner.api.container_requests().create.call_args
1622         if rev == "20220510":
1623             self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1624         else:
1625             self.assertEqual(None, kwargs['body'].get('output_properties'))
1626
1627     @parameterized.expand([
1628         ("20231117",),
1629         ("20240502",),
1630     ])
1631     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1632     def test_output_glob(self, rev, keepdocker):
1633         arvados_cwl.add_arv_hints()
1634         runner = mock.MagicMock()
1635         runner.ignore_docker_for_reuse = False
1636         runner.intermediate_output_ttl = 0
1637         runner.secret_store = cwltool.secrets.SecretStore()
1638         runner.api._rootDesc = {"revision": rev}
1639         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1640
1641         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1642         runner.api.collections().get().execute.return_value = {
1643             "portable_data_hash": "99999999999999999999999999999993+99"}
1644
1645         tool = cmap({
1646             "inputs": [{
1647                 "id": "inp",
1648                 "type": "string"
1649             }],
1650             "outputs": [
1651                 {
1652                     "id": "o1",
1653                     "type": "File",
1654                     "outputBinding": {
1655                         "glob": "*.txt"
1656                     }
1657                 },
1658                 {
1659                     "id": "o2",
1660                     "type": "File",
1661                     "outputBinding": {
1662                         "glob": ["*.dat", "*.bat"]
1663                     }
1664                 },
1665                 {
1666                     "id": "o3",
1667                     "type": {
1668                         "type": "record",
1669                         "fields": [
1670                             {
1671                                 "name": "f1",
1672                                 "type": "File",
1673                                 "outputBinding": {
1674                                     "glob": ["*.cat"]
1675                                 }
1676                             }
1677                         ]
1678                     }
1679                 },
1680                 {
1681                     "id": "o4",
1682                     "type": "File",
1683                     "outputBinding": {
1684                         "glob": "$(inputs.inp)"
1685                     }
1686                 },
1687                 {
1688                     "id": "o5",
1689                     "type": "File",
1690                     "outputBinding": {
1691                         "glob": "*.foo"
1692                     },
1693                     "secondaryFiles": [".goo", "^.hoo"]
1694                 },
1695
1696             ],
1697             "baseCommand": "ls",
1698             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1699             "id": "",
1700             "cwlVersion": "v1.2",
1701             "class": "CommandLineTool",
1702             "hints": [ ]
1703         })
1704
1705         loadingContext, runtimeContext = self.helper(runner)
1706         runtimeContext.name = "test_timelimit"
1707
1708         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1709         arvtool.formatgraph = None
1710
1711         for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1712             j.run(runtimeContext)
1713
1714         _, kwargs = runner.api.container_requests().create.call_args
1715         if rev == "20240502":
1716             self.assertEqual(['*.txt', '*.txt/**',
1717                               '*.dat', '*.dat/**',
1718                               '*.bat', '*.bat/**',
1719                               '*.cat', '*.cat/**',
1720                               'quux', 'quux/**',
1721                               '*.foo', '*.foo/**',
1722                               '*.foo.goo', '*.foo.goo/**',
1723                               '*.hoo', '*.hoo/**',
1724                               'cwl.output.json',
1725                               ], kwargs['body'].get('output_glob'))
1726         else:
1727             self.assertEqual(None, kwargs['body'].get('output_glob'))
1728
1729
1730 class TestWorkflow(unittest.TestCase):
1731     def setUp(self):
1732         cwltool.process._names = set()
1733         #arv_docker_clear_cache()
1734
1735     def helper(self, runner, enable_reuse=True):
1736         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1737
1738         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1739                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1740
1741         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1742         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1743         document_loader.fetch_text = document_loader.fetcher.fetch_text
1744         document_loader.check_exists = document_loader.fetcher.check_exists
1745
1746         loadingContext = arvados_cwl.context.ArvLoadingContext(
1747             {"avsc_names": avsc_names,
1748              "basedir": "",
1749              "make_fs_access": make_fs_access,
1750              "loader": document_loader,
1751              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1752              "construct_tool_object": runner.arv_make_tool,
1753              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__,
1754              })
1755         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1756             {"work_api": "containers",
1757              "basedir": "",
1758              "name": "test_run_wf_"+str(enable_reuse),
1759              "make_fs_access": make_fs_access,
1760              "tmpdir": "/tmp",
1761              "enable_reuse": enable_reuse,
1762              "priority": 500})
1763
1764         return loadingContext, runtimeContext
1765
1766     # The test passes no builder.resources
1767     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1768     @mock.patch("arvados.collection.CollectionReader")
1769     @mock.patch("arvados.collection.Collection")
1770     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1771     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1772         arvados_cwl.add_arv_hints()
1773
1774         api = mock.MagicMock()
1775         api._rootDesc = get_rootDesc()
1776         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1777
1778         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1779         self.assertEqual(runner.work_api, 'containers')
1780
1781         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1782         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1783         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1784                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1785
1786         runner.api.containers().current().execute.return_value = {}
1787
1788         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1789         runner.ignore_docker_for_reuse = False
1790         runner.num_retries = 0
1791         runner.secret_store = cwltool.secrets.SecretStore()
1792
1793         loadingContext, runtimeContext = self.helper(runner)
1794         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1795
1796         mockcollectionreader().exists.return_value = True
1797
1798         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1799         metadata["cwlVersion"] = tool["cwlVersion"]
1800
1801         mockc = mock.MagicMock()
1802         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1803         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1804
1805         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1806         arvtool.formatgraph = None
1807         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1808
1809         next(it).run(runtimeContext)
1810         next(it).run(runtimeContext)
1811
1812         with open("tests/wf/scatter2_subwf.cwl") as f:
1813             subwf = StripYAMLComments(f.read()).rstrip()
1814
1815         runner.api.container_requests().create.assert_called_with(
1816             body=JsonDiffMatcher({
1817                 "command": [
1818                     "cwltool",
1819                     "--no-container",
1820                     "--move-outputs",
1821                     "--preserve-entire-environment",
1822                     "workflow.cwl",
1823                     "cwl.input.yml"
1824                 ],
1825                 "container_image": "99999999999999999999999999999993+99",
1826                 "cwd": "/var/spool/cwl",
1827                 "environment": {
1828                     "HOME": "/var/spool/cwl",
1829                     "TMPDIR": "/tmp"
1830                 },
1831                 "mounts": {
1832                     "/keep/99999999999999999999999999999999+118": {
1833                         "kind": "collection",
1834                         "portable_data_hash": "99999999999999999999999999999999+118"
1835                     },
1836                     "/tmp": {
1837                         "capacity": 1073741824,
1838                         "kind": "tmp"
1839                     },
1840                     "/var/spool/cwl": {
1841                         "capacity": 1073741824,
1842                         "kind": "tmp"
1843                     },
1844                     "/var/spool/cwl/cwl.input.yml": {
1845                         "kind": "collection",
1846                         "path": "cwl.input.yml",
1847                         "portable_data_hash": "99999999999999999999999999999996+99"
1848                     },
1849                     "/var/spool/cwl/workflow.cwl": {
1850                         "kind": "collection",
1851                         "path": "workflow.cwl",
1852                         "portable_data_hash": "99999999999999999999999999999996+99"
1853                     },
1854                     "stdout": {
1855                         "kind": "file",
1856                         "path": "/var/spool/cwl/cwl.output.json"
1857                     }
1858                 },
1859                 "name": "scatterstep",
1860                 "output_name": "Output from step scatterstep",
1861                 "output_path": "/var/spool/cwl",
1862                 "output_ttl": 0,
1863                 "priority": 500,
1864                 "properties": {'cwl_input': {
1865                         "fileblub": {
1866                             "basename": "token.txt",
1867                             "class": "File",
1868                             "dirname": "/keep/99999999999999999999999999999999+118",
1869                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1870                             "nameext": ".txt",
1871                             "nameroot": "token",
1872                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1873                             "size": 0
1874                         },
1875                         "sleeptime": 5
1876                 }},
1877                 "runtime_constraints": {
1878                     "ram": 1073741824,
1879                     "vcpus": 1
1880                 },
1881                 "scheduling_parameters": {},
1882                 "secret_mounts": {},
1883                 "state": "Committed",
1884                 "use_existing": True,
1885                 'output_storage_classes': ["default"]
1886             }))
1887         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1888         mockc.open().__enter__().write.assert_has_calls([mock.call(
1889 '''{
1890   "fileblub": {
1891     "basename": "token.txt",
1892     "class": "File",
1893     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1894     "size": 0
1895   },
1896   "sleeptime": 5
1897 }''')])
1898
1899     # The test passes no builder.resources
1900     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1901     @mock.patch("arvados.collection.CollectionReader")
1902     @mock.patch("arvados.collection.Collection")
1903     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1904     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1905         arvados_cwl.add_arv_hints()
1906
1907         api = mock.MagicMock()
1908         api._rootDesc = get_rootDesc()
1909         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1910
1911         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1912         self.assertEqual(runner.work_api, 'containers')
1913
1914         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1915         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1916                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1917         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1918                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1919
1920         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1921         runner.ignore_docker_for_reuse = False
1922         runner.num_retries = 0
1923         runner.secret_store = cwltool.secrets.SecretStore()
1924
1925         loadingContext, runtimeContext = self.helper(runner)
1926         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1927         loadingContext.do_update = True
1928         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1929
1930         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1931
1932         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1933         arvtool.formatgraph = None
1934         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1935
1936         next(it).run(runtimeContext)
1937         next(it).run(runtimeContext)
1938
1939         with open("tests/wf/echo-subwf.cwl") as f:
1940             subwf = StripYAMLComments(f.read())
1941
1942         runner.api.container_requests().create.assert_called_with(
1943             body=JsonDiffMatcher({
1944                 'output_ttl': 0,
1945                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1946                 'scheduling_parameters': {},
1947                 'name': u'echo-subwf',
1948                 'secret_mounts': {},
1949                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1950                 'properties': {'cwl_input': {}},
1951                 'priority': 500,
1952                 'mounts': {
1953                     '/var/spool/cwl/cwl.input.yml': {
1954                         'portable_data_hash': '99999999999999999999999999999996+99',
1955                         'kind': 'collection',
1956                         'path': 'cwl.input.yml'
1957                     },
1958                     '/var/spool/cwl/workflow.cwl': {
1959                         'portable_data_hash': '99999999999999999999999999999996+99',
1960                         'kind': 'collection',
1961                         'path': 'workflow.cwl'
1962                     },
1963                     'stdout': {
1964                         'path': '/var/spool/cwl/cwl.output.json',
1965                         'kind': 'file'
1966                     },
1967                     '/tmp': {
1968                         'kind': 'tmp',
1969                         'capacity': 1073741824
1970                     }, '/var/spool/cwl': {
1971                         'kind': 'tmp',
1972                         'capacity': 3221225472
1973                     }
1974                 },
1975                 'state': 'Committed',
1976                 'output_path': '/var/spool/cwl',
1977                 'container_image': '99999999999999999999999999999993+99',
1978                 'command': [
1979                     u'cwltool',
1980                     u'--no-container',
1981                     u'--move-outputs',
1982                     u'--preserve-entire-environment',
1983                     u'workflow.cwl',
1984                     u'cwl.input.yml'
1985                 ],
1986                 'use_existing': True,
1987                 'output_name': u'Output from step echo-subwf',
1988                 'cwd': '/var/spool/cwl',
1989                 'output_storage_classes': ["default"]
1990             }))
1991
1992     def test_default_work_api(self):
1993         arvados_cwl.add_arv_hints()
1994
1995         api = mock.MagicMock()
1996         api._rootDesc = copy.deepcopy(get_rootDesc())
1997         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1998         self.assertEqual(runner.work_api, 'containers')