sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 import arvados_cwl.util
   8 #from arvados_cwl.arvdocker import arv_docker_clear_cache
   9 import copy
  10 import arvados.config
  11 import logging
  12 import unittest
  13 import os
  14 import functools
  15 import threading
  16 import cwltool.process
  17 import cwltool.secrets
  18 import cwltool.load_tool
  19 from cwltool.update import INTERNAL_VERSION
  20 from schema_salad.ref_resolver import Loader
  21 from schema_salad.sourceline import cmap
  22 import io
  23
  24 from unittest import mock
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         #arv_docker_clear_cache()
  64
  65     def tearDown(self):
  66         root_logger = logging.getLogger('')
  67
  68         # Remove existing RuntimeStatusLoggingHandlers if they exist
  69         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
  70         root_logger.handlers = handlers
  71
  72     def helper(self, runner, enable_reuse=True):
  73         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  74
  75         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  76                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  77         fs_access = mock.MagicMock()
  78         fs_access.exists.return_value = True
  79
  80         loadingContext = arvados_cwl.context.ArvLoadingContext(
  81             {"avsc_names": avsc_names,
  82              "basedir": "",
  83              "make_fs_access": make_fs_access,
  84              "construct_tool_object": runner.arv_make_tool,
  85              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
  86              "loader": Loader({}),
  87              "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
  88              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
  89              })
  90         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  91             {"work_api": "containers",
  92              "basedir": "",
  93              "name": "test_run_"+str(enable_reuse),
  94              "make_fs_access": make_fs_access,
  95              "tmpdir": "/tmp",
  96              "outdir": "/tmp",
  97              "enable_reuse": enable_reuse,
  98              "priority": 500,
  99              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 100              "workflow_eval_lock": threading.Condition(threading.RLock())
 101             })
 102
 103         if isinstance(runner, mock.MagicMock):
 104             def make_tool(toolpath_object, loadingContext):
 105                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
 106             runner.arv_make_tool.side_effect = make_tool
 107
 108         return loadingContext, runtimeContext
 109
 110     # Helper function to set up the ArvCwlExecutor to use the containers api
 111     # and test that the RuntimeStatusLoggingHandler is set up correctly
 112     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 113         api = mock.MagicMock()
 114         api._rootDesc = copy.deepcopy(get_rootDesc())
 115
 116         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 117         # adds the logging handler that will call runtime_status_update() mock
 118         self.assertFalse(gcc_mock.called)
 119         runner = arvados_cwl.ArvCwlExecutor(api)
 120         self.assertEqual(runner.work_api, 'containers')
 121         root_logger = logging.getLogger('')
 122         handlerClasses = [h.__class__ for h in root_logger.handlers]
 123         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 124         return runner
 125
 126     # The test passes no builder.resources
 127     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 128     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 129     def test_run(self, keepdocker):
 130         for enable_reuse in (True, False):
 131             #arv_docker_clear_cache()
 132
 133             runner = mock.MagicMock()
 134             runner.ignore_docker_for_reuse = False
 135             runner.intermediate_output_ttl = 0
 136             runner.secret_store = cwltool.secrets.SecretStore()
 137             runner.api._rootDesc = {"revision": "20210628"}
 138             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 139
 140             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 141             runner.api.collections().get().execute.return_value = {
 142                 "portable_data_hash": "99999999999999999999999999999993+99"}
 143
 144             tool = cmap({
 145                 "inputs": [],
 146                 "outputs": [],
 147                 "baseCommand": "ls",
 148                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 149                 "id": "",
 150                 "class": "CommandLineTool",
 151                 "cwlVersion": "v1.2"
 152             })
 153
 154             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 155
 156             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 157             arvtool.formatgraph = None
 158
 159             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 160                 j.run(runtimeContext)
 161                 runner.api.container_requests().create.assert_called_with(
 162                     body=JsonDiffMatcher({
 163                         'environment': {
 164                             'HOME': '/var/spool/cwl',
 165                             'TMPDIR': '/tmp'
 166                         },
 167                         'name': 'test_run_'+str(enable_reuse),
 168                         'runtime_constraints': {
 169                             'vcpus': 1,
 170                             'ram': 268435456
 171                         },
 172                         'use_existing': enable_reuse,
 173                         'priority': 500,
 174                         'mounts': {
 175                             '/tmp': {'kind': 'tmp',
 176                                      "capacity": 1073741824
 177                                  },
 178                             '/var/spool/cwl': {'kind': 'tmp',
 179                                                "capacity": 1073741824 }
 180                         },
 181                         'state': 'Committed',
 182                         'output_name': 'Output from step test_run_'+str(enable_reuse),
 183                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                         'output_path': '/var/spool/cwl',
 185                         'output_ttl': 0,
 186                         'container_image': '99999999999999999999999999999993+99',
 187                         'command': ['ls', '/var/spool/cwl'],
 188                         'cwd': '/var/spool/cwl',
 189                         'scheduling_parameters': {},
 190                         'properties': {'cwl_input': {}},
 191                         'secret_mounts': {},
 192                         'output_storage_classes': ["default"]
 193                     }))
 194
 195     # The test passes some fields in builder.resources
 196     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 197     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 198     def test_resource_requirements(self, keepdocker):
 199         arvados_cwl.add_arv_hints()
 200         runner = mock.MagicMock()
 201         runner.ignore_docker_for_reuse = False
 202         runner.intermediate_output_ttl = 3600
 203         runner.secret_store = cwltool.secrets.SecretStore()
 204         runner.api._rootDesc = {"revision": "20210628"}
 205         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 206
 207         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 208         runner.api.collections().get().execute.return_value = {
 209             "portable_data_hash": "99999999999999999999999999999993+99"}
 210
 211         tool = cmap({
 212             "inputs": [],
 213             "outputs": [],
 214             "hints": [{
 215                 "class": "ResourceRequirement",
 216                 "coresMin": 3,
 217                 "ramMin": 3000,
 218                 "tmpdirMin": 4000,
 219                 "outdirMin": 5000
 220             }, {
 221                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 222                 "keep_cache": 512
 223             }, {
 224                 "class": "http://arvados.org/cwl#APIRequirement",
 225             }, {
 226                 "class": "http://arvados.org/cwl#PartitionRequirement",
 227                 "partition": "blurb"
 228             }, {
 229                 "class": "http://arvados.org/cwl#IntermediateOutput",
 230                 "outputTTL": 7200
 231             }, {
 232                 "class": "WorkReuse",
 233                 "enableReuse": False
 234             }],
 235             "baseCommand": "ls",
 236             "id": "",
 237             "class": "CommandLineTool",
 238             "cwlVersion": "v1.2"
 239         })
 240
 241         loadingContext, runtimeContext = self.helper(runner)
 242         runtimeContext.name = "test_resource_requirements"
 243
 244         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 245         arvtool.formatgraph = None
 246         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 247             j.run(runtimeContext)
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         call_body_expected = {
 252             'environment': {
 253                 'HOME': '/var/spool/cwl',
 254                 'TMPDIR': '/tmp'
 255             },
 256             'name': 'test_resource_requirements',
 257             'runtime_constraints': {
 258                 'vcpus': 3,
 259                 'ram': 3145728000,
 260                 'keep_cache_ram': 536870912,
 261                 'API': True
 262             },
 263             'use_existing': False,
 264             'priority': 500,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 4194304000 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 5242880000 }
 270             },
 271             'state': 'Committed',
 272             'output_name': 'Output from step test_resource_requirements',
 273             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 274             'output_path': '/var/spool/cwl',
 275             'output_ttl': 7200,
 276             'container_image': '99999999999999999999999999999993+99',
 277             'command': ['ls'],
 278             'cwd': '/var/spool/cwl',
 279             'scheduling_parameters': {
 280                 'partitions': ['blurb']
 281             },
 282             'properties': {'cwl_input': {}},
 283             'secret_mounts': {},
 284             'output_storage_classes': ["default"]
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # The test passes some fields in builder.resources
 294     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 295     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 296     @mock.patch("arvados.collection.Collection")
 297     def test_initial_work_dir(self, collection_mock, keepdocker):
 298         runner = mock.MagicMock()
 299         runner.ignore_docker_for_reuse = False
 300         runner.intermediate_output_ttl = 0
 301         runner.secret_store = cwltool.secrets.SecretStore()
 302         runner.api._rootDesc = {"revision": "20210628"}
 303         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 304
 305         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 306         runner.api.collections().get().execute.return_value = {
 307             "portable_data_hash": "99999999999999999999999999999993+99"}
 308
 309         sourcemock = mock.MagicMock()
 310         def get_collection_mock(p):
 311             if "/" in p:
 312                 return (sourcemock, p.split("/", 1)[1])
 313             else:
 314                 return (sourcemock, "")
 315         runner.fs_access.get_collection.side_effect = get_collection_mock
 316
 317         vwdmock = mock.MagicMock()
 318         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 319
 320         tool = cmap({
 321             "inputs": [],
 322             "outputs": [],
 323             "hints": [{
 324                 "class": "InitialWorkDirRequirement",
 325                 "listing": [{
 326                     "class": "File",
 327                     "basename": "foo",
 328                     "location": "keep:99999999999999999999999999999995+99/bar"
 329                 },
 330                 {
 331                     "class": "Directory",
 332                     "basename": "foo2",
 333                     "location": "keep:99999999999999999999999999999995+99"
 334                 },
 335                 {
 336                     "class": "File",
 337                     "basename": "filename",
 338                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 339                 },
 340                 {
 341                     "class": "Directory",
 342                     "basename": "subdir",
 343                     "location": "keep:99999999999999999999999999999995+99/subdir"
 344                 }                        ]
 345             }],
 346             "baseCommand": "ls",
 347             "class": "CommandLineTool",
 348             "cwlVersion": "v1.2",
 349             "id": ""
 350         })
 351
 352         loadingContext, runtimeContext = self.helper(runner)
 353         runtimeContext.name = "test_initial_work_dir"
 354
 355         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 356
 357         arvtool.formatgraph = None
 358         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 359             j.run(runtimeContext)
 360
 361         call_args, call_kwargs = runner.api.container_requests().create.call_args
 362
 363         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 364         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 365         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 366         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 367
 368         call_body_expected = {
 369             'environment': {
 370                 'HOME': '/var/spool/cwl',
 371                 'TMPDIR': '/tmp'
 372             },
 373             'name': 'test_initial_work_dir',
 374             'runtime_constraints': {
 375                 'vcpus': 1,
 376                 'ram': 268435456
 377             },
 378             'use_existing': True,
 379             'priority': 500,
 380             'mounts': {
 381                 '/tmp': {'kind': 'tmp',
 382                          "capacity": 1073741824 },
 383                 '/var/spool/cwl': {'kind': 'tmp',
 384                                    "capacity": 1073741824 },
 385                 '/var/spool/cwl/foo': {
 386                     'kind': 'collection',
 387                     'path': 'foo',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 },
 390                 '/var/spool/cwl/foo2': {
 391                     'kind': 'collection',
 392                     'path': 'foo2',
 393                     'portable_data_hash': '99999999999999999999999999999996+99'
 394                 },
 395                 '/var/spool/cwl/filename': {
 396                     'kind': 'collection',
 397                     'path': 'filename',
 398                     'portable_data_hash': '99999999999999999999999999999996+99'
 399                 },
 400                 '/var/spool/cwl/subdir': {
 401                     'kind': 'collection',
 402                     'path': 'subdir',
 403                     'portable_data_hash': '99999999999999999999999999999996+99'
 404                 }
 405             },
 406             'state': 'Committed',
 407             'output_name': 'Output from step test_initial_work_dir',
 408             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409             'output_path': '/var/spool/cwl',
 410             'output_ttl': 0,
 411             'container_image': '99999999999999999999999999999993+99',
 412             'command': ['ls'],
 413             'cwd': '/var/spool/cwl',
 414             'scheduling_parameters': {
 415             },
 416             'properties': {'cwl_input': {}},
 417             'secret_mounts': {},
 418             'output_storage_classes': ["default"]
 419         }
 420
 421         call_body = call_kwargs.get('body', None)
 422         self.assertNotEqual(None, call_body)
 423         for key in call_body:
 424             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 425
 426
 427     # Test redirecting stdin/stdout/stderr
 428     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 429     def test_redirects(self, keepdocker):
 430         runner = mock.MagicMock()
 431         runner.ignore_docker_for_reuse = False
 432         runner.intermediate_output_ttl = 0
 433         runner.secret_store = cwltool.secrets.SecretStore()
 434         runner.api._rootDesc = {"revision": "20210628"}
 435         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 436
 437         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 438         runner.api.collections().get().execute.return_value = {
 439             "portable_data_hash": "99999999999999999999999999999993+99"}
 440
 441         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 442
 443         tool = cmap({
 444             "inputs": [],
 445             "outputs": [],
 446             "baseCommand": "ls",
 447             "stdout": "stdout.txt",
 448             "stderr": "stderr.txt",
 449             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 450             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 451             "id": "",
 452             "class": "CommandLineTool",
 453             "cwlVersion": "v1.2"
 454         })
 455
 456         loadingContext, runtimeContext = self.helper(runner)
 457         runtimeContext.name = "test_run_redirect"
 458
 459         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 460         arvtool.formatgraph = None
 461         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 462             j.run(runtimeContext)
 463             runner.api.container_requests().create.assert_called_with(
 464                 body=JsonDiffMatcher({
 465                     'environment': {
 466                         'HOME': '/var/spool/cwl',
 467                         'TMPDIR': '/tmp'
 468                     },
 469                     'name': 'test_run_redirect',
 470                     'runtime_constraints': {
 471                         'vcpus': 1,
 472                         'ram': 268435456
 473                     },
 474                     'use_existing': True,
 475                     'priority': 500,
 476                     'mounts': {
 477                         '/tmp': {'kind': 'tmp',
 478                                  "capacity": 1073741824 },
 479                         '/var/spool/cwl': {'kind': 'tmp',
 480                                            "capacity": 1073741824 },
 481                         "stderr": {
 482                             "kind": "file",
 483                             "path": "/var/spool/cwl/stderr.txt"
 484                         },
 485                         "stdin": {
 486                             "kind": "collection",
 487                             "path": "file.txt",
 488                             "portable_data_hash": "99999999999999999999999999999996+99"
 489                         },
 490                         "stdout": {
 491                             "kind": "file",
 492                             "path": "/var/spool/cwl/stdout.txt"
 493                         },
 494                     },
 495                     'state': 'Committed',
 496                     "output_name": "Output from step test_run_redirect",
 497                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 498                     'output_path': '/var/spool/cwl',
 499                     'output_ttl': 0,
 500                     'container_image': '99999999999999999999999999999993+99',
 501                     'command': ['ls', '/var/spool/cwl'],
 502                     'cwd': '/var/spool/cwl',
 503                     'scheduling_parameters': {},
 504                     'properties': {'cwl_input': {}},
 505                     'secret_mounts': {},
 506                     'output_storage_classes': ["default"]
 507                 }))
 508
 509     @mock.patch("arvados.collection.Collection")
 510     def test_done(self, col):
 511         api = mock.MagicMock()
 512
 513         runner = mock.MagicMock()
 514         runner.api = api
 515         runner.num_retries = 0
 516         runner.ignore_docker_for_reuse = False
 517         runner.intermediate_output_ttl = 0
 518         runner.secret_store = cwltool.secrets.SecretStore()
 519
 520         runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"}
 521
 522         runner.api.containers().get().execute.return_value = {"state":"Complete",
 523                                                               "output": "abc+123",
 524                                                               "exit_code": 0}
 525
 526         # Need to noop-out the close method otherwise it gets
 527         # discarded when closed and we can't call getvalue() to check
 528         # it.
 529         class NoopCloseStringIO(io.StringIO):
 530             def close(self):
 531                 pass
 532
 533         usage_report = NoopCloseStringIO()
 534         def colreader_action(name, mode):
 535             nonlocal usage_report
 536             if name == "node.json":
 537                 return io.StringIO("""{
 538     "ProviderType": "c5.large",
 539     "VCPUs": 2,
 540     "RAM": 4294967296,
 541     "IncludedScratch": 8000000000000,
 542     "AddedScratch": 0,
 543     "Price": 0.085,
 544     "Preemptible": false,
 545     "CUDA": {
 546         "DriverVersion": "",
 547         "HardwareCapability": "",
 548         "DeviceCount": 0
 549     }
 550 }""")
 551             if name == 'crunchstat.txt':
 552                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt")
 553             if name == 'arv-mount.txt':
 554                 return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt")
 555             if name == 'usage_report.html':
 556                 return usage_report
 557             return None
 558
 559         col().open.side_effect = colreader_action
 560         col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt']
 561
 562         loadingContext, runtimeContext = self.helper(runner)
 563
 564         arvjob = arvados_cwl.ArvadosContainer(runner,
 565                                               runtimeContext,
 566                                               [],
 567                                               mock.MagicMock(),
 568                                               {},
 569                                               None,
 570                                               [],
 571                                               [],
 572                                               "testjob")
 573         arvjob.output_callback = mock.MagicMock()
 574         arvjob.collect_outputs = mock.MagicMock()
 575         arvjob.successCodes = [0]
 576         arvjob.outdir = "/var/spool/cwl"
 577         arvjob.output_ttl = 3600
 578         arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 579
 580         arvjob.collect_outputs.return_value = {"out": "stuff"}
 581
 582         arvjob.done({
 583             "state": "Final",
 584             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 585             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 586             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 587             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 588             "modified_at": "2017-05-26T12:01:22Z",
 589             "properties": {},
 590             "name": "testjob"
 591         })
 592
 593         self.assertFalse(api.collections().create.called)
 594         self.assertFalse(runner.runtime_status_error.called)
 595
 596         # Assert that something was written to the usage report
 597         self.assertTrue(len(usage_report.getvalue()) > 0)
 598
 599         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 600         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 601         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 602
 603         runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
 604                                                                   body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
 605
 606
 607     # Test to make sure we dont call runtime_status_update if we already did
 608     # some where higher up in the call stack
 609     @mock.patch("arvados_cwl.util.get_current_container")
 610     def test_recursive_runtime_status_update(self, gcc_mock):
 611         self.setup_and_test_container_executor_and_logging(gcc_mock)
 612         root_logger = logging.getLogger('')
 613
 614         # get_current_container is invoked when we call runtime_status_update
 615         # so try and log again!
 616         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 617         try:
 618             root_logger.error("First Error")
 619         except RuntimeError:
 620             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 621
 622
 623     # Test to make sure that an exception raised from
 624     # get_current_container doesn't cause the logger to raise an
 625     # exception
 626     @mock.patch("arvados_cwl.util.get_current_container")
 627     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 628         self.setup_and_test_container_executor_and_logging(gcc_mock)
 629         root_logger = logging.getLogger('')
 630
 631         # get_current_container is invoked when we call
 632         # runtime_status_update, it is going to also raise an
 633         # exception.
 634         gcc_mock.side_effect = Exception("Second Error")
 635         try:
 636             root_logger.error("First Error")
 637         except Exception:
 638             self.fail("Exception in logger should not propagate")
 639         self.assertTrue(gcc_mock.called)
 640
 641     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 642     @mock.patch("arvados_cwl.util.get_current_container")
 643     @mock.patch("arvados.collection.CollectionReader")
 644     @mock.patch("arvados.collection.Collection")
 645     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 646         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 647
 648         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 649         self.assertTrue(gcc_mock.called)
 650
 651         runner.num_retries = 0
 652         runner.ignore_docker_for_reuse = False
 653         runner.intermediate_output_ttl = 0
 654         runner.secret_store = cwltool.secrets.SecretStore()
 655         runner.label = mock.MagicMock()
 656         runner.label.return_value = '[container testjob]'
 657
 658         runner.api.containers().get().execute.return_value = {
 659             "state":"Complete",
 660             "output": "abc+123",
 661             "exit_code": 1,
 662             "log": "def+234"
 663         }
 664
 665         col().open.return_value = []
 666
 667         loadingContext, runtimeContext = self.helper(runner)
 668
 669         arvjob = arvados_cwl.ArvadosContainer(runner,
 670                                               runtimeContext,
 671                                               [],
 672                                               mock.MagicMock(),
 673                                               {},
 674                                               None,
 675                                               [],
 676                                               [],
 677                                               "testjob")
 678         arvjob.output_callback = mock.MagicMock()
 679         arvjob.collect_outputs = mock.MagicMock()
 680         arvjob.successCodes = [0]
 681         arvjob.outdir = "/var/spool/cwl"
 682         arvjob.output_ttl = 3600
 683         arvjob.collect_outputs.return_value = {"out": "stuff"}
 684
 685         arvjob.done({
 686             "state": "Final",
 687             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 688             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 689             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 690             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 691             "modified_at": "2017-05-26T12:01:22Z",
 692             "properties": {}
 693         })
 694
 695         rts_mock.assert_has_calls([
 696             mock.call('error',
 697                       'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 698                       '  ** log is empty **'
 699                       ),
 700             mock.call('warning',
 701                       'arvados.cwl-runner: [container testjob] unable to generate resource usage report'
 702         )])
 703         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 704
 705     # The test passes no builder.resources
 706     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 707     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 708     def test_mounts(self, keepdocker):
 709         runner = mock.MagicMock()
 710         runner.ignore_docker_for_reuse = False
 711         runner.intermediate_output_ttl = 0
 712         runner.secret_store = cwltool.secrets.SecretStore()
 713         runner.api._rootDesc = {"revision": "20210628"}
 714         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 715
 716         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 717         runner.api.collections().get().execute.return_value = {
 718             "portable_data_hash": "99999999999999999999999999999994+99",
 719             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 720
 721         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 722
 723         tool = cmap({
 724             "inputs": [
 725                 {"id": "p1",
 726                  "type": "Directory"}
 727             ],
 728             "outputs": [],
 729             "baseCommand": "ls",
 730             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 731             "id": "",
 732             "class": "CommandLineTool",
 733             "cwlVersion": "v1.2"
 734         })
 735
 736         loadingContext, runtimeContext = self.helper(runner)
 737         runtimeContext.name = "test_run_mounts"
 738
 739         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 740         arvtool.formatgraph = None
 741         job_order = {
 742             "p1": {
 743                 "class": "Directory",
 744                 "location": "keep:99999999999999999999999999999994+44",
 745                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 746                 "listing": [
 747                     {
 748                         "class": "File",
 749                         "location": "keep:99999999999999999999999999999994+44/file1",
 750                     },
 751                     {
 752                         "class": "File",
 753                         "location": "keep:99999999999999999999999999999994+44/file2",
 754                     }
 755                 ]
 756             }
 757         }
 758         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 759             j.run(runtimeContext)
 760             runner.api.container_requests().create.assert_called_with(
 761                 body=JsonDiffMatcher({
 762                     'environment': {
 763                         'HOME': '/var/spool/cwl',
 764                         'TMPDIR': '/tmp'
 765                     },
 766                     'name': 'test_run_mounts',
 767                     'runtime_constraints': {
 768                         'vcpus': 1,
 769                         'ram': 268435456
 770                     },
 771                     'use_existing': True,
 772                     'priority': 500,
 773                     'mounts': {
 774                         "/keep/99999999999999999999999999999994+44": {
 775                             "kind": "collection",
 776                             "portable_data_hash": "99999999999999999999999999999994+44",
 777                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 778                         },
 779                         '/tmp': {'kind': 'tmp',
 780                                  "capacity": 1073741824 },
 781                         '/var/spool/cwl': {'kind': 'tmp',
 782                                            "capacity": 1073741824 }
 783                     },
 784                     'state': 'Committed',
 785                     'output_name': 'Output from step test_run_mounts',
 786                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 787                     'output_path': '/var/spool/cwl',
 788                     'output_ttl': 0,
 789                     'container_image': '99999999999999999999999999999994+99',
 790                     'command': ['ls', '/var/spool/cwl'],
 791                     'cwd': '/var/spool/cwl',
 792                     'scheduling_parameters': {},
 793                     'properties': {'cwl_input': {
 794                         "p1": {
 795                             "basename": "99999999999999999999999999999994+44",
 796                             "class": "Directory",
 797                             "dirname": "/keep",
 798                             "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 799                             "listing": [
 800                                 {
 801                                     "basename": "file1",
 802                                     "class": "File",
 803                                     "dirname": "/keep/99999999999999999999999999999994+44",
 804                                     "location": "keep:99999999999999999999999999999994+44/file1",
 805                                     "nameext": "",
 806                                     "nameroot": "file1",
 807                                     "path": "/keep/99999999999999999999999999999994+44/file1",
 808                                     "size": 0
 809                                 },
 810                                 {
 811                                     "basename": "file2",
 812                                     "class": "File",
 813                                     "dirname": "/keep/99999999999999999999999999999994+44",
 814                                     "location": "keep:99999999999999999999999999999994+44/file2",
 815                                     "nameext": "",
 816                                     "nameroot": "file2",
 817                                     "path": "/keep/99999999999999999999999999999994+44/file2",
 818                                     "size": 0
 819                                 }
 820                             ],
 821                             "location": "keep:99999999999999999999999999999994+44",
 822                             "path": "/keep/99999999999999999999999999999994+44"
 823                         }
 824                     }},
 825                     'secret_mounts': {},
 826                     'output_storage_classes': ["default"]
 827                 }))
 828
 829     # The test passes no builder.resources
 830     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 831     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 832     def test_secrets(self, keepdocker):
 833         arvados_cwl.add_arv_hints()
 834         runner = mock.MagicMock()
 835         runner.ignore_docker_for_reuse = False
 836         runner.intermediate_output_ttl = 0
 837         runner.secret_store = cwltool.secrets.SecretStore()
 838         runner.api._rootDesc = {"revision": "20210628"}
 839         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 840
 841         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 842         runner.api.collections().get().execute.return_value = {
 843             "portable_data_hash": "99999999999999999999999999999993+99"}
 844
 845         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 846
 847         tool = cmap({"arguments": ["md5sum", "example.conf"],
 848                      "class": "CommandLineTool",
 849                      "cwlVersion": "v1.2",
 850                      "hints": [
 851                          {
 852                              "class": "http://commonwl.org/cwltool#Secrets",
 853                              "secrets": [
 854                                  "#secret_job.cwl/pw"
 855                              ]
 856                          }
 857                      ],
 858                      "id": "",
 859                      "inputs": [
 860                          {
 861                              "id": "#secret_job.cwl/pw",
 862                              "type": "string"
 863                          }
 864                      ],
 865                      "outputs": [
 866                      ],
 867                      "requirements": [
 868                          {
 869                              "class": "InitialWorkDirRequirement",
 870                              "listing": [
 871                                  {
 872                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 873                                      "entryname": "example.conf"
 874                                  }
 875                              ]
 876                          }
 877                      ]})
 878
 879         loadingContext, runtimeContext = self.helper(runner)
 880         runtimeContext.name = "test_secrets"
 881
 882         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 883         arvtool.formatgraph = None
 884
 885         job_order = {"pw": "blorp"}
 886         runner.secret_store.store(["pw"], job_order)
 887
 888         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 889             j.run(runtimeContext)
 890             runner.api.container_requests().create.assert_called_with(
 891                 body=JsonDiffMatcher({
 892                     'environment': {
 893                         'HOME': '/var/spool/cwl',
 894                         'TMPDIR': '/tmp'
 895                     },
 896                     'name': 'test_secrets',
 897                     'runtime_constraints': {
 898                         'vcpus': 1,
 899                         'ram': 268435456
 900                     },
 901                     'use_existing': True,
 902                     'priority': 500,
 903                     'mounts': {
 904                         '/tmp': {'kind': 'tmp',
 905                                  "capacity": 1073741824
 906                              },
 907                         '/var/spool/cwl': {'kind': 'tmp',
 908                                            "capacity": 1073741824 }
 909                     },
 910                     'state': 'Committed',
 911                     'output_name': 'Output from step test_secrets',
 912                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 913                     'output_path': '/var/spool/cwl',
 914                     'output_ttl': 0,
 915                     'container_image': '99999999999999999999999999999993+99',
 916                     'command': ['md5sum', 'example.conf'],
 917                     'cwd': '/var/spool/cwl',
 918                     'scheduling_parameters': {},
 919                     'properties': {'cwl_input': job_order},
 920                     "secret_mounts": {
 921                         "/var/spool/cwl/example.conf": {
 922                             "content": "username: user\npassword: blorp\n",
 923                             "kind": "text"
 924                         }
 925                     },
 926                     'output_storage_classes': ["default"]
 927                 }))
 928
 929     # The test passes no builder.resources
 930     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 931     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 932     def test_timelimit(self, keepdocker):
 933         runner = mock.MagicMock()
 934         runner.ignore_docker_for_reuse = False
 935         runner.intermediate_output_ttl = 0
 936         runner.secret_store = cwltool.secrets.SecretStore()
 937         runner.api._rootDesc = {"revision": "20210628"}
 938         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 939
 940         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 941         runner.api.collections().get().execute.return_value = {
 942             "portable_data_hash": "99999999999999999999999999999993+99"}
 943
 944         tool = cmap({
 945             "inputs": [],
 946             "outputs": [],
 947             "baseCommand": "ls",
 948             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 949             "id": "",
 950             "cwlVersion": "v1.2",
 951             "class": "CommandLineTool",
 952             "hints": [
 953                 {
 954                     "class": "ToolTimeLimit",
 955                     "timelimit": 42
 956                 }
 957             ]
 958         })
 959
 960         loadingContext, runtimeContext = self.helper(runner)
 961         runtimeContext.name = "test_timelimit"
 962
 963         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 964         arvtool.formatgraph = None
 965
 966         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 967             j.run(runtimeContext)
 968
 969         _, kwargs = runner.api.container_requests().create.call_args
 970         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 971
 972
 973     # The test passes no builder.resources
 974     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 975     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 976     def test_setting_storage_class(self, keepdocker):
 977         #arv_docker_clear_cache()
 978
 979         runner = mock.MagicMock()
 980         runner.ignore_docker_for_reuse = False
 981         runner.intermediate_output_ttl = 0
 982         runner.secret_store = cwltool.secrets.SecretStore()
 983         runner.api._rootDesc = {"revision": "20210628"}
 984         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 985
 986         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 987         runner.api.collections().get().execute.return_value = {
 988             "portable_data_hash": "99999999999999999999999999999993+99"}
 989
 990         tool = cmap({
 991             "inputs": [],
 992             "outputs": [],
 993             "baseCommand": "ls",
 994             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 995             "id": "",
 996             "cwlVersion": "v1.2",
 997             "class": "CommandLineTool",
 998             "hints": [
 999                 {
1000                     "class": "http://arvados.org/cwl#OutputStorageClass",
1001                     "finalStorageClass": ["baz_sc", "qux_sc"],
1002                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
1003                 }
1004             ]
1005         })
1006
1007         loadingContext, runtimeContext = self.helper(runner, True)
1008
1009         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1010         arvtool.formatgraph = None
1011
1012         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1013             j.run(runtimeContext)
1014             runner.api.container_requests().create.assert_called_with(
1015                 body=JsonDiffMatcher({
1016                     'environment': {
1017                         'HOME': '/var/spool/cwl',
1018                         'TMPDIR': '/tmp'
1019                     },
1020                     'name': 'test_run_True',
1021                     'runtime_constraints': {
1022                         'vcpus': 1,
1023                         'ram': 268435456
1024                     },
1025                     'use_existing': True,
1026                     'priority': 500,
1027                     'mounts': {
1028                         '/tmp': {'kind': 'tmp',
1029                                  "capacity": 1073741824
1030                              },
1031                         '/var/spool/cwl': {'kind': 'tmp',
1032                                            "capacity": 1073741824 }
1033                     },
1034                     'state': 'Committed',
1035                     'output_name': 'Output from step test_run_True',
1036                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1037                     'output_path': '/var/spool/cwl',
1038                     'output_ttl': 0,
1039                     'container_image': '99999999999999999999999999999993+99',
1040                     'command': ['ls', '/var/spool/cwl'],
1041                     'cwd': '/var/spool/cwl',
1042                     'scheduling_parameters': {},
1043                     'properties': {'cwl_input': {}},
1044                     'secret_mounts': {},
1045                     'output_storage_classes': ["foo_sc", "bar_sc"]
1046                 }))
1047
1048
1049     # The test passes no builder.resources
1050     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1051     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1052     def test_setting_process_properties(self, keepdocker):
1053         #arv_docker_clear_cache()
1054
1055         runner = mock.MagicMock()
1056         runner.ignore_docker_for_reuse = False
1057         runner.intermediate_output_ttl = 0
1058         runner.secret_store = cwltool.secrets.SecretStore()
1059         runner.api._rootDesc = {"revision": "20210628"}
1060         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1061
1062         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1063         runner.api.collections().get().execute.return_value = {
1064             "portable_data_hash": "99999999999999999999999999999993+99"}
1065
1066         tool = cmap({
1067             "inputs": [
1068                 {"id": "x", "type": "string"}],
1069             "outputs": [],
1070             "baseCommand": "ls",
1071             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1072             "id": "",
1073             "class": "CommandLineTool",
1074             "cwlVersion": "v1.2",
1075             "hints": [
1076             {
1077                 "class": "http://arvados.org/cwl#ProcessProperties",
1078                 "processProperties": [
1079                     {"propertyName": "foo",
1080                      "propertyValue": "bar"},
1081                     {"propertyName": "baz",
1082                      "propertyValue": "$(inputs.x)"},
1083                     {"propertyName": "quux",
1084                      "propertyValue": {
1085                          "q1": 1,
1086                          "q2": 2
1087                      }
1088                     }
1089                 ],
1090             }
1091         ]
1092         })
1093
1094         loadingContext, runtimeContext = self.helper(runner, True)
1095
1096         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1097         arvtool.formatgraph = None
1098
1099         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
1100             j.run(runtimeContext)
1101             runner.api.container_requests().create.assert_called_with(
1102                 body=JsonDiffMatcher({
1103                     'environment': {
1104                         'HOME': '/var/spool/cwl',
1105                         'TMPDIR': '/tmp'
1106                     },
1107                     'name': 'test_run_True',
1108                     'runtime_constraints': {
1109                         'vcpus': 1,
1110                         'ram': 268435456
1111                     },
1112                     'use_existing': True,
1113                     'priority': 500,
1114                     'mounts': {
1115                         '/tmp': {'kind': 'tmp',
1116                                  "capacity": 1073741824
1117                              },
1118                         '/var/spool/cwl': {'kind': 'tmp',
1119                                            "capacity": 1073741824 }
1120                     },
1121                     'state': 'Committed',
1122                     'output_name': 'Output from step test_run_True',
1123                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1124                     'output_path': '/var/spool/cwl',
1125                     'output_ttl': 0,
1126                     'container_image': '99999999999999999999999999999993+99',
1127                     'command': ['ls', '/var/spool/cwl'],
1128                     'cwd': '/var/spool/cwl',
1129                     'scheduling_parameters': {},
1130                     'properties': {
1131                         "baz": "blorp",
1132                         "cwl_input": {"x": "blorp"},
1133                         "foo": "bar",
1134                         "quux": {
1135                             "q1": 1,
1136                             "q2": 2
1137                         }
1138                     },
1139                     'secret_mounts': {},
1140                     'output_storage_classes': ["default"]
1141                 }))
1142
1143
1144     # The test passes no builder.resources
1145     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1146     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1147     def test_cuda_requirement(self, keepdocker):
1148         arvados_cwl.add_arv_hints()
1149         #arv_docker_clear_cache()
1150
1151         runner = mock.MagicMock()
1152         runner.ignore_docker_for_reuse = False
1153         runner.intermediate_output_ttl = 0
1154         runner.secret_store = cwltool.secrets.SecretStore()
1155         runner.api._rootDesc = {"revision": "20210628"}
1156         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1157
1158         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1159         runner.api.collections().get().execute.return_value = {
1160             "portable_data_hash": "99999999999999999999999999999993+99"}
1161
1162         test_cwl_req = [{
1163                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1164                 "cudaVersionMin": "11.0",
1165                 "cudaComputeCapability": "9.0",
1166             }, {
1167                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1168                 "cudaVersionMin": "11.0",
1169                 "cudaComputeCapability": "9.0",
1170                 "cudaDeviceCountMin": 2
1171             }, {
1172                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1173                 "cudaVersionMin": "11.0",
1174                 "cudaComputeCapability": ["4.0", "5.0"],
1175                 "cudaDeviceCountMin": 2
1176             }]
1177
1178         test_arv_req = [{
1179             'device_count': 1,
1180             'driver_version': "11.0",
1181             'hardware_capability': "9.0"
1182         }, {
1183             'device_count': 2,
1184             'driver_version': "11.0",
1185             'hardware_capability': "9.0"
1186         }, {
1187             'device_count': 2,
1188             'driver_version': "11.0",
1189             'hardware_capability': "4.0"
1190         }]
1191
1192         for test_case in range(0, len(test_cwl_req)):
1193
1194             tool = cmap({
1195                 "inputs": [],
1196                 "outputs": [],
1197                 "baseCommand": "nvidia-smi",
1198                 "arguments": [],
1199                 "id": "",
1200                 "cwlVersion": "v1.2",
1201                 "class": "CommandLineTool",
1202                 "requirements": [test_cwl_req[test_case]]
1203             })
1204
1205             loadingContext, runtimeContext = self.helper(runner, True)
1206
1207             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1208             arvtool.formatgraph = None
1209
1210             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1211                 j.run(runtimeContext)
1212                 runner.api.container_requests().create.assert_called_with(
1213                     body=JsonDiffMatcher({
1214                         'environment': {
1215                             'HOME': '/var/spool/cwl',
1216                             'TMPDIR': '/tmp'
1217                         },
1218                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1219                         'runtime_constraints': {
1220                             'vcpus': 1,
1221                             'ram': 268435456,
1222                             'cuda': test_arv_req[test_case]
1223                         },
1224                         'use_existing': True,
1225                         'priority': 500,
1226                         'mounts': {
1227                             '/tmp': {'kind': 'tmp',
1228                                      "capacity": 1073741824
1229                                  },
1230                             '/var/spool/cwl': {'kind': 'tmp',
1231                                                "capacity": 1073741824 }
1232                         },
1233                         'state': 'Committed',
1234                         'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1235                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1236                         'output_path': '/var/spool/cwl',
1237                         'output_ttl': 0,
1238                         'container_image': '99999999999999999999999999999993+99',
1239                         'command': ['nvidia-smi'],
1240                         'cwd': '/var/spool/cwl',
1241                         'scheduling_parameters': {},
1242                         'properties': {'cwl_input': {}},
1243                         'secret_mounts': {},
1244                         'output_storage_classes': ["default"]
1245                     }))
1246
1247
1248     # The test passes no builder.resources
1249     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1250     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1251     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1252     def test_match_local_docker(self, keepdocker, determine_image_id):
1253         arvados_cwl.add_arv_hints()
1254
1255         runner = mock.MagicMock()
1256         runner.ignore_docker_for_reuse = False
1257         runner.intermediate_output_ttl = 0
1258         runner.secret_store = cwltool.secrets.SecretStore()
1259         runner.api._rootDesc = {"revision": "20210628"}
1260         runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1261
1262         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1263                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1264         determine_image_id.side_effect = lambda x: "123"
1265         def execute(uuid):
1266             ex = mock.MagicMock()
1267             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1268                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1269             ex.execute.return_value = lookup[uuid]
1270             return ex
1271         runner.api.collections().get.side_effect = execute
1272
1273         tool = cmap({
1274             "inputs": [],
1275             "outputs": [],
1276             "baseCommand": "echo",
1277             "arguments": [],
1278             "id": "",
1279             "cwlVersion": "v1.0",
1280             "class": "org.w3id.cwl.cwl.CommandLineTool"
1281         })
1282
1283         loadingContext, runtimeContext = self.helper(runner, True)
1284
1285         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
1286         arvtool.formatgraph = None
1287
1288         container_request = {
1289             'environment': {
1290                 'HOME': '/var/spool/cwl',
1291                 'TMPDIR': '/tmp'
1292             },
1293             'name': 'test_run_True',
1294             'runtime_constraints': {
1295                 'vcpus': 1,
1296                 'ram': 1073741824,
1297             },
1298             'use_existing': True,
1299             'priority': 500,
1300             'mounts': {
1301                 '/tmp': {'kind': 'tmp',
1302                          "capacity": 1073741824
1303                          },
1304                 '/var/spool/cwl': {'kind': 'tmp',
1305                                    "capacity": 1073741824 }
1306             },
1307             'state': 'Committed',
1308             'output_name': 'Output from step test_run_True',
1309             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1310             'output_path': '/var/spool/cwl',
1311             'output_ttl': 0,
1312             'container_image': '99999999999999999999999999999994+99',
1313             'command': ['echo'],
1314             'cwd': '/var/spool/cwl',
1315             'scheduling_parameters': {},
1316             'properties': {'cwl_input': {}},
1317             'secret_mounts': {},
1318             'output_storage_classes': ["default"]
1319         }
1320
1321         runtimeContext.match_local_docker = False
1322         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1323             j.run(runtimeContext)
1324             runner.api.container_requests().create.assert_called_with(
1325                 body=JsonDiffMatcher(container_request))
1326
1327         runtimeContext.cached_docker_lookups.clear()
1328         runtimeContext.match_local_docker = True
1329         container_request['container_image'] = '99999999999999999999999999999993+99'
1330         container_request['name'] = 'test_run_True_2'
1331         container_request['output_name'] = 'Output from step test_run_True_2'
1332         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1333             j.run(runtimeContext)
1334             runner.api.container_requests().create.assert_called_with(
1335                 body=JsonDiffMatcher(container_request))
1336
1337
1338     # The test passes no builder.resources
1339     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1340     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1341     def test_run_preemptible_hint(self, keepdocker):
1342         arvados_cwl.add_arv_hints()
1343         for enable_preemptible in (None, True, False):
1344             for preemptible_hint in (None, True, False):
1345                 #arv_docker_clear_cache()
1346
1347                 runner = mock.MagicMock()
1348                 runner.ignore_docker_for_reuse = False
1349                 runner.intermediate_output_ttl = 0
1350                 runner.secret_store = cwltool.secrets.SecretStore()
1351                 runner.api._rootDesc = {"revision": "20210628"}
1352                 runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1353
1354                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1355                 runner.api.collections().get().execute.return_value = {
1356                     "portable_data_hash": "99999999999999999999999999999993+99"}
1357
1358                 if preemptible_hint is not None:
1359                     hints = [{
1360                         "class": "http://arvados.org/cwl#UsePreemptible",
1361                         "usePreemptible": preemptible_hint
1362                     }]
1363                 else:
1364                     hints = []
1365
1366                 tool = cmap({
1367                     "inputs": [],
1368                     "outputs": [],
1369                     "baseCommand": "ls",
1370                     "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1371                     "id": "",
1372                     "class": "CommandLineTool",
1373                     "cwlVersion": "v1.2",
1374                     "hints": hints
1375                 })
1376
1377                 loadingContext, runtimeContext = self.helper(runner)
1378
1379                 runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
1380                 runtimeContext.enable_preemptible = enable_preemptible
1381
1382                 arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1383                 arvtool.formatgraph = None
1384
1385                 # Test the interactions between --enable/disable-preemptible
1386                 # and UsePreemptible hint
1387
1388                 if enable_preemptible is None:
1389                     if preemptible_hint is None:
1390                         sched = {}
1391                     else:
1392                         sched = {'preemptible': preemptible_hint}
1393                 else:
1394                     if preemptible_hint is None:
1395                         sched = {'preemptible': enable_preemptible}
1396                     else:
1397                         sched = {'preemptible': enable_preemptible and preemptible_hint}
1398
1399                 for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1400                     j.run(runtimeContext)
1401                     runner.api.container_requests().create.assert_called_with(
1402                         body=JsonDiffMatcher({
1403                             'environment': {
1404                                 'HOME': '/var/spool/cwl',
1405                                 'TMPDIR': '/tmp'
1406                             },
1407                             'name': runtimeContext.name,
1408                             'runtime_constraints': {
1409                                 'vcpus': 1,
1410                                 'ram': 268435456
1411                             },
1412                             'use_existing': True,
1413                             'priority': 500,
1414                             'mounts': {
1415                                 '/tmp': {'kind': 'tmp',
1416                                          "capacity": 1073741824
1417                                      },
1418                                 '/var/spool/cwl': {'kind': 'tmp',
1419                                                    "capacity": 1073741824 }
1420                             },
1421                             'state': 'Committed',
1422                             'output_name': 'Output from step '+runtimeContext.name,
1423                             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1424                             'output_path': '/var/spool/cwl',
1425                             'output_ttl': 0,
1426                             'container_image': '99999999999999999999999999999993+99',
1427                             'command': ['ls', '/var/spool/cwl'],
1428                             'cwd': '/var/spool/cwl',
1429                             'scheduling_parameters': sched,
1430                             'properties': {'cwl_input': {}},
1431                             'secret_mounts': {},
1432                             'output_storage_classes': ["default"]
1433                         }))
1434
1435
1436     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1437     def test_output_properties(self, keepdocker):
1438         arvados_cwl.add_arv_hints()
1439         for rev in ["20210628", "20220510"]:
1440             runner = mock.MagicMock()
1441             runner.ignore_docker_for_reuse = False
1442             runner.intermediate_output_ttl = 0
1443             runner.secret_store = cwltool.secrets.SecretStore()
1444             runner.api._rootDesc = {"revision": rev}
1445             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1446
1447             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1448             runner.api.collections().get().execute.return_value = {
1449                 "portable_data_hash": "99999999999999999999999999999993+99"}
1450
1451             tool = cmap({
1452                 "inputs": [{
1453                     "id": "inp",
1454                     "type": "string"
1455                 }],
1456                 "outputs": [],
1457                 "baseCommand": "ls",
1458                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1459                 "id": "",
1460                 "cwlVersion": "v1.2",
1461                 "class": "CommandLineTool",
1462                 "hints": [
1463                     {
1464                         "class": "http://arvados.org/cwl#OutputCollectionProperties",
1465                         "outputProperties": {
1466                             "foo": "bar",
1467                             "baz": "$(inputs.inp)"
1468                         }
1469                     }
1470                 ]
1471             })
1472
1473             loadingContext, runtimeContext = self.helper(runner)
1474             runtimeContext.name = "test_timelimit"
1475
1476             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1477             arvtool.formatgraph = None
1478
1479             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1480                 j.run(runtimeContext)
1481
1482             _, kwargs = runner.api.container_requests().create.call_args
1483             if rev == "20220510":
1484                 self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties'))
1485             else:
1486                 self.assertEqual(None, kwargs['body'].get('output_properties'))
1487
1488     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1489     def test_output_glob(self, keepdocker):
1490         arvados_cwl.add_arv_hints()
1491         for rev in ["20231117", "20240502"]:
1492             runner = mock.MagicMock()
1493             runner.ignore_docker_for_reuse = False
1494             runner.intermediate_output_ttl = 0
1495             runner.secret_store = cwltool.secrets.SecretStore()
1496             runner.api._rootDesc = {"revision": rev}
1497             runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1498
1499             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1500             runner.api.collections().get().execute.return_value = {
1501                 "portable_data_hash": "99999999999999999999999999999993+99"}
1502
1503             tool = cmap({
1504                 "inputs": [{
1505                     "id": "inp",
1506                     "type": "string"
1507                 }],
1508                 "outputs": [
1509                     {
1510                         "id": "o1",
1511                         "type": "File",
1512                         "outputBinding": {
1513                             "glob": "*.txt"
1514                         }
1515                     },
1516                     {
1517                         "id": "o2",
1518                         "type": "File",
1519                         "outputBinding": {
1520                             "glob": ["*.dat", "*.bat"]
1521                         }
1522                     },
1523                     {
1524                         "id": "o3",
1525                         "type": {
1526                             "type": "record",
1527                             "fields": [
1528                                 {
1529                                     "name": "f1",
1530                                     "type": "File",
1531                                     "outputBinding": {
1532                                         "glob": ["*.cat"]
1533                                     }
1534                                 }
1535                             ]
1536                         }
1537                     },
1538                     {
1539                         "id": "o4",
1540                         "type": "File",
1541                         "outputBinding": {
1542                             "glob": "$(inputs.inp)"
1543                         }
1544                     },
1545                     {
1546                         "id": "o5",
1547                         "type": "File",
1548                         "outputBinding": {
1549                             "glob": "*.foo"
1550                         },
1551                         "secondaryFiles": [".goo", "^.hoo"]
1552                     },
1553
1554                 ],
1555                 "baseCommand": "ls",
1556                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
1557                 "id": "",
1558                 "cwlVersion": "v1.2",
1559                 "class": "CommandLineTool",
1560                 "hints": [ ]
1561             })
1562
1563             loadingContext, runtimeContext = self.helper(runner)
1564             runtimeContext.name = "test_timelimit"
1565
1566             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1567             arvtool.formatgraph = None
1568
1569             for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext):
1570                 j.run(runtimeContext)
1571
1572             _, kwargs = runner.api.container_requests().create.call_args
1573             if rev == "20240502":
1574                 self.assertEqual(['*.txt', '*.txt/**',
1575                                   '*.dat', '*.dat/**',
1576                                   '*.bat', '*.bat/**',
1577                                   '*.cat', '*.cat/**',
1578                                   'quux', 'quux/**',
1579                                   '*.foo', '*.foo/**',
1580                                   '*.foo.goo', '*.foo.goo/**',
1581                                   '*.hoo', '*.hoo/**',
1582                                   'cwl.output.json',
1583                                   ], kwargs['body'].get('output_glob'))
1584             else:
1585                 self.assertEqual(None, kwargs['body'].get('output_glob'))
1586
1587
1588 class TestWorkflow(unittest.TestCase):
1589     def setUp(self):
1590         cwltool.process._names = set()
1591         #arv_docker_clear_cache()
1592
1593     def helper(self, runner, enable_reuse=True):
1594         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1595
1596         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1597                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1598
1599         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1600         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1601         document_loader.fetch_text = document_loader.fetcher.fetch_text
1602         document_loader.check_exists = document_loader.fetcher.check_exists
1603
1604         loadingContext = arvados_cwl.context.ArvLoadingContext(
1605             {"avsc_names": avsc_names,
1606              "basedir": "",
1607              "make_fs_access": make_fs_access,
1608              "loader": document_loader,
1609              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1610              "construct_tool_object": runner.arv_make_tool,
1611              "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
1612         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1613             {"work_api": "containers",
1614              "basedir": "",
1615              "name": "test_run_wf_"+str(enable_reuse),
1616              "make_fs_access": make_fs_access,
1617              "tmpdir": "/tmp",
1618              "enable_reuse": enable_reuse,
1619              "priority": 500})
1620
1621         return loadingContext, runtimeContext
1622
1623     # The test passes no builder.resources
1624     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1625     @mock.patch("arvados.collection.CollectionReader")
1626     @mock.patch("arvados.collection.Collection")
1627     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1628     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1629         arvados_cwl.add_arv_hints()
1630
1631         api = mock.MagicMock()
1632         api._rootDesc = get_rootDesc()
1633         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1634
1635         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1636         self.assertEqual(runner.work_api, 'containers')
1637
1638         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1639         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1640         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1641                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1642
1643         runner.api.containers().current().execute.return_value = {}
1644
1645         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1646         runner.ignore_docker_for_reuse = False
1647         runner.num_retries = 0
1648         runner.secret_store = cwltool.secrets.SecretStore()
1649
1650         loadingContext, runtimeContext = self.helper(runner)
1651         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1652
1653         mockcollectionreader().exists.return_value = True
1654
1655         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1656         metadata["cwlVersion"] = tool["cwlVersion"]
1657
1658         mockc = mock.MagicMock()
1659         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1660         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1661
1662         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1663         arvtool.formatgraph = None
1664         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1665
1666         next(it).run(runtimeContext)
1667         next(it).run(runtimeContext)
1668
1669         with open("tests/wf/scatter2_subwf.cwl") as f:
1670             subwf = StripYAMLComments(f.read()).rstrip()
1671
1672         runner.api.container_requests().create.assert_called_with(
1673             body=JsonDiffMatcher({
1674                 "command": [
1675                     "cwltool",
1676                     "--no-container",
1677                     "--move-outputs",
1678                     "--preserve-entire-environment",
1679                     "workflow.cwl",
1680                     "cwl.input.yml"
1681                 ],
1682                 "container_image": "99999999999999999999999999999993+99",
1683                 "cwd": "/var/spool/cwl",
1684                 "environment": {
1685                     "HOME": "/var/spool/cwl",
1686                     "TMPDIR": "/tmp"
1687                 },
1688                 "mounts": {
1689                     "/keep/99999999999999999999999999999999+118": {
1690                         "kind": "collection",
1691                         "portable_data_hash": "99999999999999999999999999999999+118"
1692                     },
1693                     "/tmp": {
1694                         "capacity": 1073741824,
1695                         "kind": "tmp"
1696                     },
1697                     "/var/spool/cwl": {
1698                         "capacity": 1073741824,
1699                         "kind": "tmp"
1700                     },
1701                     "/var/spool/cwl/cwl.input.yml": {
1702                         "kind": "collection",
1703                         "path": "cwl.input.yml",
1704                         "portable_data_hash": "99999999999999999999999999999996+99"
1705                     },
1706                     "/var/spool/cwl/workflow.cwl": {
1707                         "kind": "collection",
1708                         "path": "workflow.cwl",
1709                         "portable_data_hash": "99999999999999999999999999999996+99"
1710                     },
1711                     "stdout": {
1712                         "kind": "file",
1713                         "path": "/var/spool/cwl/cwl.output.json"
1714                     }
1715                 },
1716                 "name": "scatterstep",
1717                 "output_name": "Output from step scatterstep",
1718                 "output_path": "/var/spool/cwl",
1719                 "output_ttl": 0,
1720                 "priority": 500,
1721                 "properties": {'cwl_input': {
1722                         "fileblub": {
1723                             "basename": "token.txt",
1724                             "class": "File",
1725                             "dirname": "/keep/99999999999999999999999999999999+118",
1726                             "location": "keep:99999999999999999999999999999999+118/token.txt",
1727                             "nameext": ".txt",
1728                             "nameroot": "token",
1729                             "path": "/keep/99999999999999999999999999999999+118/token.txt",
1730                             "size": 0
1731                         },
1732                         "sleeptime": 5
1733                 }},
1734                 "runtime_constraints": {
1735                     "ram": 1073741824,
1736                     "vcpus": 1
1737                 },
1738                 "scheduling_parameters": {},
1739                 "secret_mounts": {},
1740                 "state": "Committed",
1741                 "use_existing": True,
1742                 'output_storage_classes': ["default"]
1743             }))
1744         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1745         mockc.open().__enter__().write.assert_has_calls([mock.call(
1746 '''{
1747   "fileblub": {
1748     "basename": "token.txt",
1749     "class": "File",
1750     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1751     "size": 0
1752   },
1753   "sleeptime": 5
1754 }''')])
1755
1756     # The test passes no builder.resources
1757     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1758     @mock.patch("arvados.collection.CollectionReader")
1759     @mock.patch("arvados.collection.Collection")
1760     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1761     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1762         arvados_cwl.add_arv_hints()
1763
1764         api = mock.MagicMock()
1765         api._rootDesc = get_rootDesc()
1766         api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
1767
1768         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1769         self.assertEqual(runner.work_api, 'containers')
1770
1771         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1772         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1773                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1774         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1775                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1776
1777         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1778         runner.ignore_docker_for_reuse = False
1779         runner.num_retries = 0
1780         runner.secret_store = cwltool.secrets.SecretStore()
1781
1782         loadingContext, runtimeContext = self.helper(runner)
1783         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1784         loadingContext.do_update = True
1785         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1786
1787         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1788
1789         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1790         arvtool.formatgraph = None
1791         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1792
1793         next(it).run(runtimeContext)
1794         next(it).run(runtimeContext)
1795
1796         with open("tests/wf/echo-subwf.cwl") as f:
1797             subwf = StripYAMLComments(f.read())
1798
1799         runner.api.container_requests().create.assert_called_with(
1800             body=JsonDiffMatcher({
1801                 'output_ttl': 0,
1802                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1803                 'scheduling_parameters': {},
1804                 'name': u'echo-subwf',
1805                 'secret_mounts': {},
1806                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1807                 'properties': {'cwl_input': {}},
1808                 'priority': 500,
1809                 'mounts': {
1810                     '/var/spool/cwl/cwl.input.yml': {
1811                         'portable_data_hash': '99999999999999999999999999999996+99',
1812                         'kind': 'collection',
1813                         'path': 'cwl.input.yml'
1814                     },
1815                     '/var/spool/cwl/workflow.cwl': {
1816                         'portable_data_hash': '99999999999999999999999999999996+99',
1817                         'kind': 'collection',
1818                         'path': 'workflow.cwl'
1819                     },
1820                     'stdout': {
1821                         'path': '/var/spool/cwl/cwl.output.json',
1822                         'kind': 'file'
1823                     },
1824                     '/tmp': {
1825                         'kind': 'tmp',
1826                         'capacity': 1073741824
1827                     }, '/var/spool/cwl': {
1828                         'kind': 'tmp',
1829                         'capacity': 3221225472
1830                     }
1831                 },
1832                 'state': 'Committed',
1833                 'output_path': '/var/spool/cwl',
1834                 'container_image': '99999999999999999999999999999993+99',
1835                 'command': [
1836                     u'cwltool',
1837                     u'--no-container',
1838                     u'--move-outputs',
1839                     u'--preserve-entire-environment',
1840                     u'workflow.cwl',
1841                     u'cwl.input.yml'
1842                 ],
1843                 'use_existing': True,
1844                 'output_name': u'Output from step echo-subwf',
1845                 'cwd': '/var/spool/cwl',
1846                 'output_storage_classes': ["default"]
1847             }))
1848
1849     def test_default_work_api(self):
1850         arvados_cwl.add_arv_hints()
1851
1852         api = mock.MagicMock()
1853         api._rootDesc = copy.deepcopy(get_rootDesc())
1854         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1855         self.assertEqual(runner.work_api, 'containers')