sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 from arvados_cwl.arvdocker import arv_docker_clear_cache
   8 import copy
   9 import arvados.config
  10 import logging
  11 import mock
  12 import unittest
  13 import os
  14 import functools
  15 import cwltool.process
  16 import cwltool.secrets
  17 from schema_salad.ref_resolver import Loader
  18 from schema_salad.sourceline import cmap
  19
  20 from .matcher import JsonDiffMatcher
  21 from .mock_discovery import get_rootDesc
  22
  23 if not os.getenv('ARVADOS_DEBUG'):
  24     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  25     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  26
  27 class CollectionMock(object):
  28     def __init__(self, vwdmock, *args, **kwargs):
  29         self.vwdmock = vwdmock
  30         self.count = 0
  31
  32     def open(self, *args, **kwargs):
  33         self.count += 1
  34         return self.vwdmock.open(*args, **kwargs)
  35
  36     def copy(self, *args, **kwargs):
  37         self.count += 1
  38         self.vwdmock.copy(*args, **kwargs)
  39
  40     def save_new(self, *args, **kwargs):
  41         pass
  42
  43     def __len__(self):
  44         return self.count
  45
  46     def portable_data_hash(self):
  47         if self.count == 0:
  48             return arvados.config.EMPTY_BLOCK_LOCATOR
  49         else:
  50             return "99999999999999999999999999999996+99"
  51
  52
  53 class TestContainer(unittest.TestCase):
  54
  55     def helper(self, runner, enable_reuse=True):
  56         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  57
  58         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  59                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  60         loadingContext = arvados_cwl.context.ArvLoadingContext(
  61             {"avsc_names": avsc_names,
  62              "basedir": "",
  63              "make_fs_access": make_fs_access,
  64              "loader": Loader({}),
  65              "metadata": {"cwlVersion": "v1.0"}})
  66         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  67             {"work_api": "containers",
  68              "basedir": "",
  69              "name": "test_run_"+str(enable_reuse),
  70              "make_fs_access": make_fs_access,
  71              "tmpdir": "/tmp",
  72              "enable_reuse": enable_reuse,
  73              "priority": 500})
  74
  75         return loadingContext, runtimeContext
  76
  77     # The test passes no builder.resources
  78     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  79     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  80     def test_run(self, keepdocker):
  81         for enable_reuse in (True, False):
  82             arv_docker_clear_cache()
  83
  84             runner = mock.MagicMock()
  85             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  86             runner.ignore_docker_for_reuse = False
  87             runner.intermediate_output_ttl = 0
  88             runner.secret_store = cwltool.secrets.SecretStore()
  89
  90             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  91             runner.api.collections().get().execute.return_value = {
  92                 "portable_data_hash": "99999999999999999999999999999993+99"}
  93
  94             tool = cmap({
  95                 "inputs": [],
  96                 "outputs": [],
  97                 "baseCommand": "ls",
  98                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  99                 "id": "#",
 100                 "class": "CommandLineTool"
 101             })
 102
 103             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 104
 105             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 106             arvtool.formatgraph = None
 107
 108             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 109                 j.run(runtimeContext)
 110                 runner.api.container_requests().create.assert_called_with(
 111                     body=JsonDiffMatcher({
 112                         'environment': {
 113                             'HOME': '/var/spool/cwl',
 114                             'TMPDIR': '/tmp'
 115                         },
 116                         'name': 'test_run_'+str(enable_reuse),
 117                         'runtime_constraints': {
 118                             'vcpus': 1,
 119                             'ram': 1073741824
 120                         },
 121                         'use_existing': enable_reuse,
 122                         'priority': 500,
 123                         'mounts': {
 124                             '/tmp': {'kind': 'tmp',
 125                                      "capacity": 1073741824
 126                                  },
 127                             '/var/spool/cwl': {'kind': 'tmp',
 128                                                "capacity": 1073741824 }
 129                         },
 130                         'state': 'Committed',
 131                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 132                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 133                         'output_path': '/var/spool/cwl',
 134                         'output_ttl': 0,
 135                         'container_image': 'arvados/jobs',
 136                         'command': ['ls', '/var/spool/cwl'],
 137                         'cwd': '/var/spool/cwl',
 138                         'scheduling_parameters': {},
 139                         'properties': {},
 140                         'secret_mounts': {}
 141                     }))
 142
 143     # The test passes some fields in builder.resources
 144     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 145     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 146     def test_resource_requirements(self, keepdocker):
 147         arv_docker_clear_cache()
 148         runner = mock.MagicMock()
 149         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 150         runner.ignore_docker_for_reuse = False
 151         runner.intermediate_output_ttl = 3600
 152         runner.secret_store = cwltool.secrets.SecretStore()
 153
 154         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 155         runner.api.collections().get().execute.return_value = {
 156             "portable_data_hash": "99999999999999999999999999999993+99"}
 157
 158         tool = cmap({
 159             "inputs": [],
 160             "outputs": [],
 161             "hints": [{
 162                 "class": "ResourceRequirement",
 163                 "coresMin": 3,
 164                 "ramMin": 3000,
 165                 "tmpdirMin": 4000,
 166                 "outdirMin": 5000
 167             }, {
 168                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 169                 "keep_cache": 512
 170             }, {
 171                 "class": "http://arvados.org/cwl#APIRequirement",
 172             }, {
 173                 "class": "http://arvados.org/cwl#PartitionRequirement",
 174                 "partition": "blurb"
 175             }, {
 176                 "class": "http://arvados.org/cwl#IntermediateOutput",
 177                 "outputTTL": 7200
 178             }, {
 179                 "class": "http://arvados.org/cwl#ReuseRequirement",
 180                 "enableReuse": False
 181             }],
 182             "baseCommand": "ls",
 183             "id": "#",
 184             "class": "CommandLineTool"
 185         })
 186
 187         loadingContext, runtimeContext = self.helper(runner)
 188         runtimeContext.name = "test_resource_requirements"
 189
 190         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 191         arvtool.formatgraph = None
 192         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 193             j.run(runtimeContext)
 194
 195         call_args, call_kwargs = runner.api.container_requests().create.call_args
 196
 197         call_body_expected = {
 198             'environment': {
 199                 'HOME': '/var/spool/cwl',
 200                 'TMPDIR': '/tmp'
 201             },
 202             'name': 'test_resource_requirements',
 203             'runtime_constraints': {
 204                 'vcpus': 3,
 205                 'ram': 3145728000,
 206                 'keep_cache_ram': 536870912,
 207                 'API': True
 208             },
 209             'use_existing': False,
 210             'priority': 500,
 211             'mounts': {
 212                 '/tmp': {'kind': 'tmp',
 213                          "capacity": 4194304000 },
 214                 '/var/spool/cwl': {'kind': 'tmp',
 215                                    "capacity": 5242880000 }
 216             },
 217             'state': 'Committed',
 218             'output_name': 'Output for step test_resource_requirements',
 219             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 220             'output_path': '/var/spool/cwl',
 221             'output_ttl': 7200,
 222             'container_image': 'arvados/jobs',
 223             'command': ['ls'],
 224             'cwd': '/var/spool/cwl',
 225             'scheduling_parameters': {
 226                 'partitions': ['blurb']
 227             },
 228             'properties': {},
 229             'secret_mounts': {}
 230         }
 231
 232         call_body = call_kwargs.get('body', None)
 233         self.assertNotEqual(None, call_body)
 234         for key in call_body:
 235             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 236
 237
 238     # The test passes some fields in builder.resources
 239     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 240     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 241     @mock.patch("arvados.collection.Collection")
 242     def test_initial_work_dir(self, collection_mock, keepdocker):
 243         arv_docker_clear_cache()
 244         runner = mock.MagicMock()
 245         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 246         runner.ignore_docker_for_reuse = False
 247         runner.intermediate_output_ttl = 0
 248         runner.secret_store = cwltool.secrets.SecretStore()
 249
 250         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 251         runner.api.collections().get().execute.return_value = {
 252             "portable_data_hash": "99999999999999999999999999999993+99"}
 253
 254         sourcemock = mock.MagicMock()
 255         def get_collection_mock(p):
 256             if "/" in p:
 257                 return (sourcemock, p.split("/", 1)[1])
 258             else:
 259                 return (sourcemock, "")
 260         runner.fs_access.get_collection.side_effect = get_collection_mock
 261
 262         vwdmock = mock.MagicMock()
 263         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 264
 265         tool = cmap({
 266             "inputs": [],
 267             "outputs": [],
 268             "hints": [{
 269                 "class": "InitialWorkDirRequirement",
 270                 "listing": [{
 271                     "class": "File",
 272                     "basename": "foo",
 273                     "location": "keep:99999999999999999999999999999995+99/bar"
 274                 },
 275                 {
 276                     "class": "Directory",
 277                     "basename": "foo2",
 278                     "location": "keep:99999999999999999999999999999995+99"
 279                 },
 280                 {
 281                     "class": "File",
 282                     "basename": "filename",
 283                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 284                 },
 285                 {
 286                     "class": "Directory",
 287                     "basename": "subdir",
 288                     "location": "keep:99999999999999999999999999999995+99/subdir"
 289                 }                        ]
 290             }],
 291             "baseCommand": "ls",
 292             "id": "#",
 293             "class": "CommandLineTool"
 294         })
 295
 296         loadingContext, runtimeContext = self.helper(runner)
 297         runtimeContext.name = "test_initial_work_dir"
 298
 299         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 300         arvtool.formatgraph = None
 301         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 302             j.run(runtimeContext)
 303
 304         call_args, call_kwargs = runner.api.container_requests().create.call_args
 305
 306         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 307         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 308         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 309         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 310
 311         call_body_expected = {
 312             'environment': {
 313                 'HOME': '/var/spool/cwl',
 314                 'TMPDIR': '/tmp'
 315             },
 316             'name': 'test_initial_work_dir',
 317             'runtime_constraints': {
 318                 'vcpus': 1,
 319                 'ram': 1073741824
 320             },
 321             'use_existing': True,
 322             'priority': 500,
 323             'mounts': {
 324                 '/tmp': {'kind': 'tmp',
 325                          "capacity": 1073741824 },
 326                 '/var/spool/cwl': {'kind': 'tmp',
 327                                    "capacity": 1073741824 },
 328                 '/var/spool/cwl/foo': {
 329                     'kind': 'collection',
 330                     'path': 'foo',
 331                     'portable_data_hash': '99999999999999999999999999999996+99'
 332                 },
 333                 '/var/spool/cwl/foo2': {
 334                     'kind': 'collection',
 335                     'path': 'foo2',
 336                     'portable_data_hash': '99999999999999999999999999999996+99'
 337                 },
 338                 '/var/spool/cwl/filename': {
 339                     'kind': 'collection',
 340                     'path': 'filename',
 341                     'portable_data_hash': '99999999999999999999999999999996+99'
 342                 },
 343                 '/var/spool/cwl/subdir': {
 344                     'kind': 'collection',
 345                     'path': 'subdir',
 346                     'portable_data_hash': '99999999999999999999999999999996+99'
 347                 }
 348             },
 349             'state': 'Committed',
 350             'output_name': 'Output for step test_initial_work_dir',
 351             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 352             'output_path': '/var/spool/cwl',
 353             'output_ttl': 0,
 354             'container_image': 'arvados/jobs',
 355             'command': ['ls'],
 356             'cwd': '/var/spool/cwl',
 357             'scheduling_parameters': {
 358             },
 359             'properties': {},
 360             'secret_mounts': {}
 361         }
 362
 363         call_body = call_kwargs.get('body', None)
 364         self.assertNotEqual(None, call_body)
 365         for key in call_body:
 366             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 367
 368
 369     # Test redirecting stdin/stdout/stderr
 370     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 371     def test_redirects(self, keepdocker):
 372         arv_docker_clear_cache()
 373
 374         runner = mock.MagicMock()
 375         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 376         runner.ignore_docker_for_reuse = False
 377         runner.intermediate_output_ttl = 0
 378         runner.secret_store = cwltool.secrets.SecretStore()
 379
 380         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 381         runner.api.collections().get().execute.return_value = {
 382             "portable_data_hash": "99999999999999999999999999999993+99"}
 383
 384         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 385
 386         tool = cmap({
 387             "inputs": [],
 388             "outputs": [],
 389             "baseCommand": "ls",
 390             "stdout": "stdout.txt",
 391             "stderr": "stderr.txt",
 392             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 393             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 394             "id": "#",
 395             "class": "CommandLineTool"
 396         })
 397
 398         loadingContext, runtimeContext = self.helper(runner)
 399         runtimeContext.name = "test_run_redirect"
 400
 401         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 402         arvtool.formatgraph = None
 403         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 404             j.run(runtimeContext)
 405             runner.api.container_requests().create.assert_called_with(
 406                 body=JsonDiffMatcher({
 407                     'environment': {
 408                         'HOME': '/var/spool/cwl',
 409                         'TMPDIR': '/tmp'
 410                     },
 411                     'name': 'test_run_redirect',
 412                     'runtime_constraints': {
 413                         'vcpus': 1,
 414                         'ram': 1073741824
 415                     },
 416                     'use_existing': True,
 417                     'priority': 500,
 418                     'mounts': {
 419                         '/tmp': {'kind': 'tmp',
 420                                  "capacity": 1073741824 },
 421                         '/var/spool/cwl': {'kind': 'tmp',
 422                                            "capacity": 1073741824 },
 423                         "stderr": {
 424                             "kind": "file",
 425                             "path": "/var/spool/cwl/stderr.txt"
 426                         },
 427                         "stdin": {
 428                             "kind": "collection",
 429                             "path": "file.txt",
 430                             "portable_data_hash": "99999999999999999999999999999996+99"
 431                         },
 432                         "stdout": {
 433                             "kind": "file",
 434                             "path": "/var/spool/cwl/stdout.txt"
 435                         },
 436                     },
 437                     'state': 'Committed',
 438                     "output_name": "Output for step test_run_redirect",
 439                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 440                     'output_path': '/var/spool/cwl',
 441                     'output_ttl': 0,
 442                     'container_image': 'arvados/jobs',
 443                     'command': ['ls', '/var/spool/cwl'],
 444                     'cwd': '/var/spool/cwl',
 445                     'scheduling_parameters': {},
 446                     'properties': {},
 447                     'secret_mounts': {}
 448                 }))
 449
 450     @mock.patch("arvados.collection.Collection")
 451     def test_done(self, col):
 452         api = mock.MagicMock()
 453
 454         runner = mock.MagicMock()
 455         runner.api = api
 456         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 457         runner.num_retries = 0
 458         runner.ignore_docker_for_reuse = False
 459         runner.intermediate_output_ttl = 0
 460         runner.secret_store = cwltool.secrets.SecretStore()
 461
 462         runner.api.containers().get().execute.return_value = {"state":"Complete",
 463                                                               "output": "abc+123",
 464                                                               "exit_code": 0}
 465
 466         col().open.return_value = []
 467
 468         arvjob = arvados_cwl.ArvadosContainer(runner,
 469                                               mock.MagicMock(),
 470                                               {},
 471                                               None,
 472                                               [],
 473                                               [],
 474                                               "testjob")
 475         arvjob.output_callback = mock.MagicMock()
 476         arvjob.collect_outputs = mock.MagicMock()
 477         arvjob.successCodes = [0]
 478         arvjob.outdir = "/var/spool/cwl"
 479         arvjob.output_ttl = 3600
 480
 481         arvjob.collect_outputs.return_value = {"out": "stuff"}
 482
 483         arvjob.done({
 484             "state": "Final",
 485             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 486             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 487             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 488             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 489             "modified_at": "2017-05-26T12:01:22Z"
 490         })
 491
 492         self.assertFalse(api.collections().create.called)
 493         self.assertFalse(runner.runtime_status_error.called)
 494
 495         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 496         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 497         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 498
 499     @mock.patch("arvados_cwl.get_current_container")
 500     @mock.patch("arvados.collection.CollectionReader")
 501     @mock.patch("arvados.collection.Collection")
 502     def test_child_failure(self, col, reader, gcc_mock):
 503         api = mock.MagicMock()
 504         api._rootDesc = copy.deepcopy(get_rootDesc())
 505         del api._rootDesc.get('resources')['jobs']['methods']['create']
 506
 507         # Set up runner with mocked runtime_status_update()
 508         self.assertFalse(gcc_mock.called)
 509         runtime_status_update = mock.MagicMock()
 510         arvados_cwl.ArvCwlRunner.runtime_status_update = runtime_status_update
 511         runner = arvados_cwl.ArvCwlRunner(api)
 512         self.assertEqual(runner.work_api, 'containers')
 513
 514         # Make sure ArvCwlRunner thinks it's running inside a container so it
 515         # adds the logging handler that will call runtime_status_update() mock
 516         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 517         self.assertTrue(gcc_mock.called)
 518         root_logger = logging.getLogger('')
 519         handlerClasses = [h.__class__ for h in root_logger.handlers]
 520         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 521
 522         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 523         runner.num_retries = 0
 524         runner.ignore_docker_for_reuse = False
 525         runner.intermediate_output_ttl = 0
 526         runner.secret_store = cwltool.secrets.SecretStore()
 527         runner.label = mock.MagicMock()
 528         runner.label.return_value = '[container testjob]'
 529
 530         runner.api.containers().get().execute.return_value = {
 531             "state":"Complete",
 532             "output": "abc+123",
 533             "exit_code": 1,
 534             "log": "def+234"
 535         }
 536
 537         col().open.return_value = []
 538
 539         arvjob = arvados_cwl.ArvadosContainer(runner,
 540                                               mock.MagicMock(),
 541                                               {},
 542                                               None,
 543                                               [],
 544                                               [],
 545                                               "testjob")
 546         arvjob.output_callback = mock.MagicMock()
 547         arvjob.collect_outputs = mock.MagicMock()
 548         arvjob.successCodes = [0]
 549         arvjob.outdir = "/var/spool/cwl"
 550         arvjob.output_ttl = 3600
 551         arvjob.collect_outputs.return_value = {"out": "stuff"}
 552
 553         arvjob.done({
 554             "state": "Final",
 555             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 556             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 557             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 558             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 559             "modified_at": "2017-05-26T12:01:22Z"
 560         })
 561
 562         runtime_status_update.assert_called_with(
 563             'error',
 564             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 565             '  ** log is empty **'
 566         )
 567         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 568
 569     # The test passes no builder.resources
 570     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 571     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 572     def test_mounts(self, keepdocker):
 573         arv_docker_clear_cache()
 574
 575         runner = mock.MagicMock()
 576         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 577         runner.ignore_docker_for_reuse = False
 578         runner.intermediate_output_ttl = 0
 579         runner.secret_store = cwltool.secrets.SecretStore()
 580
 581         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 582         runner.api.collections().get().execute.return_value = {
 583             "portable_data_hash": "99999999999999999999999999999994+99",
 584             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 585
 586         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 587
 588         tool = cmap({
 589             "inputs": [
 590                 {"id": "p1",
 591                  "type": "Directory"}
 592             ],
 593             "outputs": [],
 594             "baseCommand": "ls",
 595             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 596             "id": "#",
 597             "class": "CommandLineTool"
 598         })
 599
 600         loadingContext, runtimeContext = self.helper(runner)
 601         runtimeContext.name = "test_run_mounts"
 602
 603         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 604         arvtool.formatgraph = None
 605         job_order = {
 606             "p1": {
 607                 "class": "Directory",
 608                 "location": "keep:99999999999999999999999999999994+44",
 609                 "listing": [
 610                     {
 611                         "class": "File",
 612                         "location": "keep:99999999999999999999999999999994+44/file1",
 613                     },
 614                     {
 615                         "class": "File",
 616                         "location": "keep:99999999999999999999999999999994+44/file2",
 617                     }
 618                 ]
 619             }
 620         }
 621         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 622             j.run(runtimeContext)
 623             runner.api.container_requests().create.assert_called_with(
 624                 body=JsonDiffMatcher({
 625                     'environment': {
 626                         'HOME': '/var/spool/cwl',
 627                         'TMPDIR': '/tmp'
 628                     },
 629                     'name': 'test_run_mounts',
 630                     'runtime_constraints': {
 631                         'vcpus': 1,
 632                         'ram': 1073741824
 633                     },
 634                     'use_existing': True,
 635                     'priority': 500,
 636                     'mounts': {
 637                         "/keep/99999999999999999999999999999994+44": {
 638                             "kind": "collection",
 639                             "portable_data_hash": "99999999999999999999999999999994+44"
 640                         },
 641                         '/tmp': {'kind': 'tmp',
 642                                  "capacity": 1073741824 },
 643                         '/var/spool/cwl': {'kind': 'tmp',
 644                                            "capacity": 1073741824 }
 645                     },
 646                     'state': 'Committed',
 647                     'output_name': 'Output for step test_run_mounts',
 648                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 649                     'output_path': '/var/spool/cwl',
 650                     'output_ttl': 0,
 651                     'container_image': 'arvados/jobs',
 652                     'command': ['ls', '/var/spool/cwl'],
 653                     'cwd': '/var/spool/cwl',
 654                     'scheduling_parameters': {},
 655                     'properties': {},
 656                     'secret_mounts': {}
 657                 }))
 658
 659     # The test passes no builder.resources
 660     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 661     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 662     def test_secrets(self, keepdocker):
 663         arv_docker_clear_cache()
 664
 665         runner = mock.MagicMock()
 666         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 667         runner.ignore_docker_for_reuse = False
 668         runner.intermediate_output_ttl = 0
 669         runner.secret_store = cwltool.secrets.SecretStore()
 670
 671         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 672         runner.api.collections().get().execute.return_value = {
 673             "portable_data_hash": "99999999999999999999999999999993+99"}
 674
 675         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 676
 677         tool = cmap({"arguments": ["md5sum", "example.conf"],
 678                      "class": "CommandLineTool",
 679                      "hints": [
 680                          {
 681                              "class": "http://commonwl.org/cwltool#Secrets",
 682                              "secrets": [
 683                                  "#secret_job.cwl/pw"
 684                              ]
 685                          }
 686                      ],
 687                      "id": "#secret_job.cwl",
 688                      "inputs": [
 689                          {
 690                              "id": "#secret_job.cwl/pw",
 691                              "type": "string"
 692                          }
 693                      ],
 694                      "outputs": [
 695                      ],
 696                      "requirements": [
 697                          {
 698                              "class": "InitialWorkDirRequirement",
 699                              "listing": [
 700                                  {
 701                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 702                                      "entryname": "example.conf"
 703                                  }
 704                              ]
 705                          }
 706                      ]})
 707
 708         loadingContext, runtimeContext = self.helper(runner)
 709         runtimeContext.name = "test_secrets"
 710
 711         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 712         arvtool.formatgraph = None
 713
 714         job_order = {"pw": "blorp"}
 715         runner.secret_store.store(["pw"], job_order)
 716
 717         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 718             j.run(runtimeContext)
 719             runner.api.container_requests().create.assert_called_with(
 720                 body=JsonDiffMatcher({
 721                     'environment': {
 722                         'HOME': '/var/spool/cwl',
 723                         'TMPDIR': '/tmp'
 724                     },
 725                     'name': 'test_secrets',
 726                     'runtime_constraints': {
 727                         'vcpus': 1,
 728                         'ram': 1073741824
 729                     },
 730                     'use_existing': True,
 731                     'priority': 500,
 732                     'mounts': {
 733                         '/tmp': {'kind': 'tmp',
 734                                  "capacity": 1073741824
 735                              },
 736                         '/var/spool/cwl': {'kind': 'tmp',
 737                                            "capacity": 1073741824 }
 738                     },
 739                     'state': 'Committed',
 740                     'output_name': 'Output for step test_secrets',
 741                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 742                     'output_path': '/var/spool/cwl',
 743                     'output_ttl': 0,
 744                     'container_image': 'arvados/jobs',
 745                     'command': ['md5sum', 'example.conf'],
 746                     'cwd': '/var/spool/cwl',
 747                     'scheduling_parameters': {},
 748                     'properties': {},
 749                     "secret_mounts": {
 750                         "/var/spool/cwl/example.conf": {
 751                             "content": "username: user\npassword: blorp\n",
 752                             "kind": "text"
 753                         }
 754                     }
 755                 }))
 756
 757     # The test passes no builder.resources
 758     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 759     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 760     def test_timelimit(self, keepdocker):
 761         arv_docker_clear_cache()
 762
 763         runner = mock.MagicMock()
 764         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 765         runner.ignore_docker_for_reuse = False
 766         runner.intermediate_output_ttl = 0
 767         runner.secret_store = cwltool.secrets.SecretStore()
 768
 769         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 770         runner.api.collections().get().execute.return_value = {
 771             "portable_data_hash": "99999999999999999999999999999993+99"}
 772
 773         tool = cmap({
 774             "inputs": [],
 775             "outputs": [],
 776             "baseCommand": "ls",
 777             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 778             "id": "#",
 779             "class": "CommandLineTool",
 780             "hints": [
 781                 {
 782                     "class": "http://commonwl.org/cwltool#TimeLimit",
 783                     "timelimit": 42
 784                 }
 785             ]
 786         })
 787
 788         loadingContext, runtimeContext = self.helper(runner)
 789         runtimeContext.name = "test_timelimit"
 790
 791         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 792         arvtool.formatgraph = None
 793
 794         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 795             j.run(runtimeContext)
 796
 797         _, kwargs = runner.api.container_requests().create.call_args
 798         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))