sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 from arvados_cwl.arvdocker import arv_docker_clear_cache
   8 import copy
   9 import arvados.config
  10 import logging
  11 import mock
  12 import unittest
  13 import os
  14 import functools
  15 import cwltool.process
  16 import cwltool.secrets
  17 from schema_salad.ref_resolver import Loader
  18 from schema_salad.sourceline import cmap
  19
  20 from .matcher import JsonDiffMatcher
  21 from .mock_discovery import get_rootDesc
  22
  23 if not os.getenv('ARVADOS_DEBUG'):
  24     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  25     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  26
  27 class CollectionMock(object):
  28     def __init__(self, vwdmock, *args, **kwargs):
  29         self.vwdmock = vwdmock
  30         self.count = 0
  31
  32     def open(self, *args, **kwargs):
  33         self.count += 1
  34         return self.vwdmock.open(*args, **kwargs)
  35
  36     def copy(self, *args, **kwargs):
  37         self.count += 1
  38         self.vwdmock.copy(*args, **kwargs)
  39
  40     def save_new(self, *args, **kwargs):
  41         pass
  42
  43     def __len__(self):
  44         return self.count
  45
  46     def portable_data_hash(self):
  47         if self.count == 0:
  48             return arvados.config.EMPTY_BLOCK_LOCATOR
  49         else:
  50             return "99999999999999999999999999999996+99"
  51
  52
  53 class TestContainer(unittest.TestCase):
  54
  55     def helper(self, runner, enable_reuse=True):
  56         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  57
  58         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  59                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  60         loadingContext = arvados_cwl.context.ArvLoadingContext(
  61             {"avsc_names": avsc_names,
  62              "basedir": "",
  63              "make_fs_access": make_fs_access,
  64              "loader": Loader({}),
  65              "metadata": {"cwlVersion": "v1.0"}})
  66         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  67             {"work_api": "containers",
  68              "basedir": "",
  69              "name": "test_run_"+str(enable_reuse),
  70              "make_fs_access": make_fs_access,
  71              "tmpdir": "/tmp",
  72              "enable_reuse": enable_reuse,
  73              "priority": 500})
  74
  75         return loadingContext, runtimeContext
  76
  77     # The test passes no builder.resources
  78     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  79     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  80     def test_run(self, keepdocker):
  81         for enable_reuse in (True, False):
  82             arv_docker_clear_cache()
  83
  84             runner = mock.MagicMock()
  85             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  86             runner.ignore_docker_for_reuse = False
  87             runner.intermediate_output_ttl = 0
  88             runner.secret_store = cwltool.secrets.SecretStore()
  89
  90             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  91             runner.api.collections().get().execute.return_value = {
  92                 "portable_data_hash": "99999999999999999999999999999993+99"}
  93
  94             tool = cmap({
  95                 "inputs": [],
  96                 "outputs": [],
  97                 "baseCommand": "ls",
  98                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  99                 "id": "#",
 100                 "class": "CommandLineTool"
 101             })
 102
 103             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 104
 105             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 106             arvtool.formatgraph = None
 107
 108             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 109                 j.run(runtimeContext)
 110                 runner.api.container_requests().create.assert_called_with(
 111                     body=JsonDiffMatcher({
 112                         'environment': {
 113                             'HOME': '/var/spool/cwl',
 114                             'TMPDIR': '/tmp'
 115                         },
 116                         'name': 'test_run_'+str(enable_reuse),
 117                         'runtime_constraints': {
 118                             'vcpus': 1,
 119                             'ram': 1073741824
 120                         },
 121                         'use_existing': enable_reuse,
 122                         'priority': 500,
 123                         'mounts': {
 124                             '/tmp': {'kind': 'tmp',
 125                                      "capacity": 1073741824
 126                                  },
 127                             '/var/spool/cwl': {'kind': 'tmp',
 128                                                "capacity": 1073741824 }
 129                         },
 130                         'state': 'Committed',
 131                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 132                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 133                         'output_path': '/var/spool/cwl',
 134                         'output_ttl': 0,
 135                         'container_image': 'arvados/jobs',
 136                         'command': ['ls', '/var/spool/cwl'],
 137                         'cwd': '/var/spool/cwl',
 138                         'scheduling_parameters': {},
 139                         'properties': {},
 140                         'secret_mounts': {}
 141                     }))
 142
 143     # The test passes some fields in builder.resources
 144     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 145     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 146     def test_resource_requirements(self, keepdocker):
 147         arv_docker_clear_cache()
 148         runner = mock.MagicMock()
 149         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 150         runner.ignore_docker_for_reuse = False
 151         runner.intermediate_output_ttl = 3600
 152         runner.secret_store = cwltool.secrets.SecretStore()
 153
 154         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 155         runner.api.collections().get().execute.return_value = {
 156             "portable_data_hash": "99999999999999999999999999999993+99"}
 157
 158         tool = cmap({
 159             "inputs": [],
 160             "outputs": [],
 161             "hints": [{
 162                 "class": "ResourceRequirement",
 163                 "coresMin": 3,
 164                 "ramMin": 3000,
 165                 "tmpdirMin": 4000,
 166                 "outdirMin": 5000
 167             }, {
 168                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 169                 "keep_cache": 512
 170             }, {
 171                 "class": "http://arvados.org/cwl#APIRequirement",
 172             }, {
 173                 "class": "http://arvados.org/cwl#PartitionRequirement",
 174                 "partition": "blurb"
 175             }, {
 176                 "class": "http://arvados.org/cwl#IntermediateOutput",
 177                 "outputTTL": 7200
 178             }, {
 179                 "class": "http://arvados.org/cwl#ReuseRequirement",
 180                 "enableReuse": False
 181             }],
 182             "baseCommand": "ls",
 183             "id": "#",
 184             "class": "CommandLineTool"
 185         })
 186
 187         loadingContext, runtimeContext = self.helper(runner)
 188         runtimeContext.name = "test_resource_requirements"
 189
 190         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 191         arvtool.formatgraph = None
 192         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 193             j.run(runtimeContext)
 194
 195         call_args, call_kwargs = runner.api.container_requests().create.call_args
 196
 197         call_body_expected = {
 198             'environment': {
 199                 'HOME': '/var/spool/cwl',
 200                 'TMPDIR': '/tmp'
 201             },
 202             'name': 'test_resource_requirements',
 203             'runtime_constraints': {
 204                 'vcpus': 3,
 205                 'ram': 3145728000,
 206                 'keep_cache_ram': 536870912,
 207                 'API': True
 208             },
 209             'use_existing': False,
 210             'priority': 500,
 211             'mounts': {
 212                 '/tmp': {'kind': 'tmp',
 213                          "capacity": 4194304000 },
 214                 '/var/spool/cwl': {'kind': 'tmp',
 215                                    "capacity": 5242880000 }
 216             },
 217             'state': 'Committed',
 218             'output_name': 'Output for step test_resource_requirements',
 219             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 220             'output_path': '/var/spool/cwl',
 221             'output_ttl': 7200,
 222             'container_image': 'arvados/jobs',
 223             'command': ['ls'],
 224             'cwd': '/var/spool/cwl',
 225             'scheduling_parameters': {
 226                 'partitions': ['blurb']
 227             },
 228             'properties': {},
 229             'secret_mounts': {}
 230         }
 231
 232         call_body = call_kwargs.get('body', None)
 233         self.assertNotEqual(None, call_body)
 234         for key in call_body:
 235             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 236
 237
 238     # The test passes some fields in builder.resources
 239     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 240     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 241     @mock.patch("arvados.collection.Collection")
 242     def test_initial_work_dir(self, collection_mock, keepdocker):
 243         arv_docker_clear_cache()
 244         runner = mock.MagicMock()
 245         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 246         runner.ignore_docker_for_reuse = False
 247         runner.intermediate_output_ttl = 0
 248         runner.secret_store = cwltool.secrets.SecretStore()
 249
 250         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 251         runner.api.collections().get().execute.return_value = {
 252             "portable_data_hash": "99999999999999999999999999999993+99"}
 253
 254         sourcemock = mock.MagicMock()
 255         def get_collection_mock(p):
 256             if "/" in p:
 257                 return (sourcemock, p.split("/", 1)[1])
 258             else:
 259                 return (sourcemock, "")
 260         runner.fs_access.get_collection.side_effect = get_collection_mock
 261
 262         vwdmock = mock.MagicMock()
 263         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 264
 265         tool = cmap({
 266             "inputs": [],
 267             "outputs": [],
 268             "hints": [{
 269                 "class": "InitialWorkDirRequirement",
 270                 "listing": [{
 271                     "class": "File",
 272                     "basename": "foo",
 273                     "location": "keep:99999999999999999999999999999995+99/bar"
 274                 },
 275                 {
 276                     "class": "Directory",
 277                     "basename": "foo2",
 278                     "location": "keep:99999999999999999999999999999995+99"
 279                 },
 280                 {
 281                     "class": "File",
 282                     "basename": "filename",
 283                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 284                 },
 285                 {
 286                     "class": "Directory",
 287                     "basename": "subdir",
 288                     "location": "keep:99999999999999999999999999999995+99/subdir"
 289                 }                        ]
 290             }],
 291             "baseCommand": "ls",
 292             "id": "#",
 293             "class": "CommandLineTool"
 294         })
 295
 296         loadingContext, runtimeContext = self.helper(runner)
 297         runtimeContext.name = "test_initial_work_dir"
 298
 299         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 300         arvtool.formatgraph = None
 301         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 302             j.run(runtimeContext)
 303
 304         call_args, call_kwargs = runner.api.container_requests().create.call_args
 305
 306         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 307         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 308         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 309         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 310
 311         call_body_expected = {
 312             'environment': {
 313                 'HOME': '/var/spool/cwl',
 314                 'TMPDIR': '/tmp'
 315             },
 316             'name': 'test_initial_work_dir',
 317             'runtime_constraints': {
 318                 'vcpus': 1,
 319                 'ram': 1073741824
 320             },
 321             'use_existing': True,
 322             'priority': 500,
 323             'mounts': {
 324                 '/tmp': {'kind': 'tmp',
 325                          "capacity": 1073741824 },
 326                 '/var/spool/cwl': {'kind': 'tmp',
 327                                    "capacity": 1073741824 },
 328                 '/var/spool/cwl/foo': {
 329                     'kind': 'collection',
 330                     'path': 'foo',
 331                     'portable_data_hash': '99999999999999999999999999999996+99'
 332                 },
 333                 '/var/spool/cwl/foo2': {
 334                     'kind': 'collection',
 335                     'path': 'foo2',
 336                     'portable_data_hash': '99999999999999999999999999999996+99'
 337                 },
 338                 '/var/spool/cwl/filename': {
 339                     'kind': 'collection',
 340                     'path': 'filename',
 341                     'portable_data_hash': '99999999999999999999999999999996+99'
 342                 },
 343                 '/var/spool/cwl/subdir': {
 344                     'kind': 'collection',
 345                     'path': 'subdir',
 346                     'portable_data_hash': '99999999999999999999999999999996+99'
 347                 }
 348             },
 349             'state': 'Committed',
 350             'output_name': 'Output for step test_initial_work_dir',
 351             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 352             'output_path': '/var/spool/cwl',
 353             'output_ttl': 0,
 354             'container_image': 'arvados/jobs',
 355             'command': ['ls'],
 356             'cwd': '/var/spool/cwl',
 357             'scheduling_parameters': {
 358             },
 359             'properties': {},
 360             'secret_mounts': {}
 361         }
 362
 363         call_body = call_kwargs.get('body', None)
 364         self.assertNotEqual(None, call_body)
 365         for key in call_body:
 366             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 367
 368
 369     # Test redirecting stdin/stdout/stderr
 370     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 371     def test_redirects(self, keepdocker):
 372         arv_docker_clear_cache()
 373
 374         runner = mock.MagicMock()
 375         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 376         runner.ignore_docker_for_reuse = False
 377         runner.intermediate_output_ttl = 0
 378         runner.secret_store = cwltool.secrets.SecretStore()
 379
 380         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 381         runner.api.collections().get().execute.return_value = {
 382             "portable_data_hash": "99999999999999999999999999999993+99"}
 383
 384         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 385
 386         tool = cmap({
 387             "inputs": [],
 388             "outputs": [],
 389             "baseCommand": "ls",
 390             "stdout": "stdout.txt",
 391             "stderr": "stderr.txt",
 392             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 393             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 394             "id": "#",
 395             "class": "CommandLineTool"
 396         })
 397
 398         loadingContext, runtimeContext = self.helper(runner)
 399         runtimeContext.name = "test_run_redirect"
 400
 401         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 402         arvtool.formatgraph = None
 403         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 404             j.run(runtimeContext)
 405             runner.api.container_requests().create.assert_called_with(
 406                 body=JsonDiffMatcher({
 407                     'environment': {
 408                         'HOME': '/var/spool/cwl',
 409                         'TMPDIR': '/tmp'
 410                     },
 411                     'name': 'test_run_redirect',
 412                     'runtime_constraints': {
 413                         'vcpus': 1,
 414                         'ram': 1073741824
 415                     },
 416                     'use_existing': True,
 417                     'priority': 500,
 418                     'mounts': {
 419                         '/tmp': {'kind': 'tmp',
 420                                  "capacity": 1073741824 },
 421                         '/var/spool/cwl': {'kind': 'tmp',
 422                                            "capacity": 1073741824 },
 423                         "stderr": {
 424                             "kind": "file",
 425                             "path": "/var/spool/cwl/stderr.txt"
 426                         },
 427                         "stdin": {
 428                             "kind": "collection",
 429                             "path": "file.txt",
 430                             "portable_data_hash": "99999999999999999999999999999996+99"
 431                         },
 432                         "stdout": {
 433                             "kind": "file",
 434                             "path": "/var/spool/cwl/stdout.txt"
 435                         },
 436                     },
 437                     'state': 'Committed',
 438                     "output_name": "Output for step test_run_redirect",
 439                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 440                     'output_path': '/var/spool/cwl',
 441                     'output_ttl': 0,
 442                     'container_image': 'arvados/jobs',
 443                     'command': ['ls', '/var/spool/cwl'],
 444                     'cwd': '/var/spool/cwl',
 445                     'scheduling_parameters': {},
 446                     'properties': {},
 447                     'secret_mounts': {}
 448                 }))
 449
 450     @mock.patch("arvados.collection.Collection")
 451     def test_done(self, col):
 452         api = mock.MagicMock()
 453
 454         runner = mock.MagicMock()
 455         runner.api = api
 456         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 457         runner.num_retries = 0
 458         runner.ignore_docker_for_reuse = False
 459         runner.intermediate_output_ttl = 0
 460         runner.secret_store = cwltool.secrets.SecretStore()
 461
 462         runner.api.containers().get().execute.return_value = {"state":"Complete",
 463                                                               "output": "abc+123",
 464                                                               "exit_code": 0}
 465
 466         col().open.return_value = []
 467
 468         arvjob = arvados_cwl.ArvadosContainer(runner,
 469                                               mock.MagicMock(),
 470                                               {},
 471                                               None,
 472                                               [],
 473                                               [],
 474                                               "testjob")
 475         arvjob.output_callback = mock.MagicMock()
 476         arvjob.collect_outputs = mock.MagicMock()
 477         arvjob.successCodes = [0]
 478         arvjob.outdir = "/var/spool/cwl"
 479         arvjob.output_ttl = 3600
 480
 481         arvjob.collect_outputs.return_value = {"out": "stuff"}
 482
 483         arvjob.done({
 484             "state": "Final",
 485             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 486             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 487             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 488             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 489             "modified_at": "2017-05-26T12:01:22Z"
 490         })
 491
 492         self.assertFalse(api.collections().create.called)
 493         self.assertFalse(runner.runtime_status_error.called)
 494
 495         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 496         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 497         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 498
 499     @mock.patch("arvados_cwl.get_current_container")
 500     @mock.patch("arvados.collection.CollectionReader")
 501     @mock.patch("arvados.collection.Collection")
 502     def test_child_failure(self, col, reader, gcc_mock):
 503         api = mock.MagicMock()
 504         api._rootDesc = copy.deepcopy(get_rootDesc())
 505         del api._rootDesc.get('resources')['jobs']['methods']['create']
 506
 507         # Set up runner with mocked runtime_status_update()
 508         self.assertFalse(gcc_mock.called)
 509         runtime_status_update = mock.MagicMock()
 510         arvados_cwl.ArvCwlRunner.runtime_status_update = runtime_status_update
 511         runner = arvados_cwl.ArvCwlRunner(api)
 512         self.assertEqual(runner.work_api, 'containers')
 513
 514         # Make sure ArvCwlRunner thinks it's running inside a container so it
 515         # adds the logging handler that will call runtime_status_update() mock
 516         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 517         self.assertTrue(gcc_mock.called)
 518         root_logger = logging.getLogger('')
 519         self.assertEqual(1, len(root_logger.handlers))
 520         handler = root_logger.handlers[0]
 521         self.assertEqual(arvados_cwl.RuntimeStatusLoggingHandler, handler.__class__)
 522
 523         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 524         runner.num_retries = 0
 525         runner.ignore_docker_for_reuse = False
 526         runner.intermediate_output_ttl = 0
 527         runner.secret_store = cwltool.secrets.SecretStore()
 528         runner.label = mock.MagicMock()
 529         runner.label.return_value = '[container testjob]'
 530
 531         runner.api.containers().get().execute.return_value = {
 532             "state":"Complete",
 533             "output": "abc+123",
 534             "exit_code": 1,
 535             "log": "def+234"
 536         }
 537
 538         col().open.return_value = []
 539
 540         arvjob = arvados_cwl.ArvadosContainer(runner,
 541                                               mock.MagicMock(),
 542                                               {},
 543                                               None,
 544                                               [],
 545                                               [],
 546                                               "testjob")
 547         arvjob.output_callback = mock.MagicMock()
 548         arvjob.collect_outputs = mock.MagicMock()
 549         arvjob.successCodes = [0]
 550         arvjob.outdir = "/var/spool/cwl"
 551         arvjob.output_ttl = 3600
 552         arvjob.collect_outputs.return_value = {"out": "stuff"}
 553
 554         arvjob.done({
 555             "state": "Final",
 556             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 557             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 558             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 559             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 560             "modified_at": "2017-05-26T12:01:22Z"
 561         })
 562
 563         runtime_status_update.assert_called_with(
 564             'error',
 565             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 566             '  ** log is empty **'
 567         )
 568         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 569
 570     # The test passes no builder.resources
 571     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 572     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 573     def test_mounts(self, keepdocker):
 574         arv_docker_clear_cache()
 575
 576         runner = mock.MagicMock()
 577         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 578         runner.ignore_docker_for_reuse = False
 579         runner.intermediate_output_ttl = 0
 580         runner.secret_store = cwltool.secrets.SecretStore()
 581
 582         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 583         runner.api.collections().get().execute.return_value = {
 584             "portable_data_hash": "99999999999999999999999999999994+99",
 585             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 586
 587         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 588
 589         tool = cmap({
 590             "inputs": [
 591                 {"id": "p1",
 592                  "type": "Directory"}
 593             ],
 594             "outputs": [],
 595             "baseCommand": "ls",
 596             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 597             "id": "#",
 598             "class": "CommandLineTool"
 599         })
 600
 601         loadingContext, runtimeContext = self.helper(runner)
 602         runtimeContext.name = "test_run_mounts"
 603
 604         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 605         arvtool.formatgraph = None
 606         job_order = {
 607             "p1": {
 608                 "class": "Directory",
 609                 "location": "keep:99999999999999999999999999999994+44",
 610                 "listing": [
 611                     {
 612                         "class": "File",
 613                         "location": "keep:99999999999999999999999999999994+44/file1",
 614                     },
 615                     {
 616                         "class": "File",
 617                         "location": "keep:99999999999999999999999999999994+44/file2",
 618                     }
 619                 ]
 620             }
 621         }
 622         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 623             j.run(runtimeContext)
 624             runner.api.container_requests().create.assert_called_with(
 625                 body=JsonDiffMatcher({
 626                     'environment': {
 627                         'HOME': '/var/spool/cwl',
 628                         'TMPDIR': '/tmp'
 629                     },
 630                     'name': 'test_run_mounts',
 631                     'runtime_constraints': {
 632                         'vcpus': 1,
 633                         'ram': 1073741824
 634                     },
 635                     'use_existing': True,
 636                     'priority': 500,
 637                     'mounts': {
 638                         "/keep/99999999999999999999999999999994+44": {
 639                             "kind": "collection",
 640                             "portable_data_hash": "99999999999999999999999999999994+44"
 641                         },
 642                         '/tmp': {'kind': 'tmp',
 643                                  "capacity": 1073741824 },
 644                         '/var/spool/cwl': {'kind': 'tmp',
 645                                            "capacity": 1073741824 }
 646                     },
 647                     'state': 'Committed',
 648                     'output_name': 'Output for step test_run_mounts',
 649                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 650                     'output_path': '/var/spool/cwl',
 651                     'output_ttl': 0,
 652                     'container_image': 'arvados/jobs',
 653                     'command': ['ls', '/var/spool/cwl'],
 654                     'cwd': '/var/spool/cwl',
 655                     'scheduling_parameters': {},
 656                     'properties': {},
 657                     'secret_mounts': {}
 658                 }))
 659
 660     # The test passes no builder.resources
 661     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 662     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 663     def test_secrets(self, keepdocker):
 664         arv_docker_clear_cache()
 665
 666         runner = mock.MagicMock()
 667         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 668         runner.ignore_docker_for_reuse = False
 669         runner.intermediate_output_ttl = 0
 670         runner.secret_store = cwltool.secrets.SecretStore()
 671
 672         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 673         runner.api.collections().get().execute.return_value = {
 674             "portable_data_hash": "99999999999999999999999999999993+99"}
 675
 676         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 677
 678         tool = cmap({"arguments": ["md5sum", "example.conf"],
 679                      "class": "CommandLineTool",
 680                      "hints": [
 681                          {
 682                              "class": "http://commonwl.org/cwltool#Secrets",
 683                              "secrets": [
 684                                  "#secret_job.cwl/pw"
 685                              ]
 686                          }
 687                      ],
 688                      "id": "#secret_job.cwl",
 689                      "inputs": [
 690                          {
 691                              "id": "#secret_job.cwl/pw",
 692                              "type": "string"
 693                          }
 694                      ],
 695                      "outputs": [
 696                      ],
 697                      "requirements": [
 698                          {
 699                              "class": "InitialWorkDirRequirement",
 700                              "listing": [
 701                                  {
 702                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 703                                      "entryname": "example.conf"
 704                                  }
 705                              ]
 706                          }
 707                      ]})
 708
 709         loadingContext, runtimeContext = self.helper(runner)
 710         runtimeContext.name = "test_secrets"
 711
 712         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 713         arvtool.formatgraph = None
 714
 715         job_order = {"pw": "blorp"}
 716         runner.secret_store.store(["pw"], job_order)
 717
 718         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 719             j.run(runtimeContext)
 720             runner.api.container_requests().create.assert_called_with(
 721                 body=JsonDiffMatcher({
 722                     'environment': {
 723                         'HOME': '/var/spool/cwl',
 724                         'TMPDIR': '/tmp'
 725                     },
 726                     'name': 'test_secrets',
 727                     'runtime_constraints': {
 728                         'vcpus': 1,
 729                         'ram': 1073741824
 730                     },
 731                     'use_existing': True,
 732                     'priority': 500,
 733                     'mounts': {
 734                         '/tmp': {'kind': 'tmp',
 735                                  "capacity": 1073741824
 736                              },
 737                         '/var/spool/cwl': {'kind': 'tmp',
 738                                            "capacity": 1073741824 }
 739                     },
 740                     'state': 'Committed',
 741                     'output_name': 'Output for step test_secrets',
 742                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 743                     'output_path': '/var/spool/cwl',
 744                     'output_ttl': 0,
 745                     'container_image': 'arvados/jobs',
 746                     'command': ['md5sum', 'example.conf'],
 747                     'cwd': '/var/spool/cwl',
 748                     'scheduling_parameters': {},
 749                     'properties': {},
 750                     "secret_mounts": {
 751                         "/var/spool/cwl/example.conf": {
 752                             "content": "username: user\npassword: blorp\n",
 753                             "kind": "text"
 754                         }
 755                     }
 756                 }))
 757
 758     # The test passes no builder.resources
 759     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 760     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 761     def test_timelimit(self, keepdocker):
 762         arv_docker_clear_cache()
 763
 764         runner = mock.MagicMock()
 765         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 766         runner.ignore_docker_for_reuse = False
 767         runner.intermediate_output_ttl = 0
 768         runner.secret_store = cwltool.secrets.SecretStore()
 769
 770         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 771         runner.api.collections().get().execute.return_value = {
 772             "portable_data_hash": "99999999999999999999999999999993+99"}
 773
 774         tool = cmap({
 775             "inputs": [],
 776             "outputs": [],
 777             "baseCommand": "ls",
 778             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 779             "id": "#",
 780             "class": "CommandLineTool",
 781             "hints": [
 782                 {
 783                     "class": "http://commonwl.org/cwltool#TimeLimit",
 784                     "timelimit": 42
 785                 }
 786             ]
 787         })
 788
 789         loadingContext, runtimeContext = self.helper(runner)
 790         runtimeContext.name = "test_timelimit"
 791
 792         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 793         arvtool.formatgraph = None
 794
 795         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 796             j.run(runtimeContext)
 797
 798         _, kwargs = runner.api.container_requests().create.call_args
 799         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))