sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 from arvados_cwl.arvdocker import arv_docker_clear_cache
   8 import arvados.config
   9 import logging
  10 import mock
  11 import unittest
  12 import os
  13 import functools
  14 import cwltool.process
  15 import cwltool.secrets
  16 from schema_salad.ref_resolver import Loader
  17 from schema_salad.sourceline import cmap
  18
  19 from .matcher import JsonDiffMatcher
  20
  21 if not os.getenv('ARVADOS_DEBUG'):
  22     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  23     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  24
  25 class CollectionMock(object):
  26     def __init__(self, vwdmock, *args, **kwargs):
  27         self.vwdmock = vwdmock
  28         self.count = 0
  29
  30     def open(self, *args, **kwargs):
  31         self.count += 1
  32         return self.vwdmock.open(*args, **kwargs)
  33
  34     def copy(self, *args, **kwargs):
  35         self.count += 1
  36         self.vwdmock.copy(*args, **kwargs)
  37
  38     def save_new(self, *args, **kwargs):
  39         pass
  40
  41     def __len__(self):
  42         return self.count
  43
  44     def portable_data_hash(self):
  45         if self.count == 0:
  46             return arvados.config.EMPTY_BLOCK_LOCATOR
  47         else:
  48             return "99999999999999999999999999999996+99"
  49
  50
  51 class TestContainer(unittest.TestCase):
  52
  53     def helper(self, runner, enable_reuse=True):
  54         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  55
  56         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  57                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  58         loadingContext = arvados_cwl.context.ArvLoadingContext(
  59             {"avsc_names": avsc_names,
  60              "basedir": "",
  61              "make_fs_access": make_fs_access,
  62              "loader": Loader({}),
  63              "metadata": {"cwlVersion": "v1.0"}})
  64         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  65             {"work_api": "containers",
  66              "basedir": "",
  67              "name": "test_run_"+str(enable_reuse),
  68              "make_fs_access": make_fs_access,
  69              "tmpdir": "/tmp",
  70              "enable_reuse": enable_reuse,
  71              "priority": 500})
  72
  73         return loadingContext, runtimeContext
  74
  75     # The test passes no builder.resources
  76     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  77     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  78     def test_run(self, keepdocker):
  79         for enable_reuse in (True, False):
  80             arv_docker_clear_cache()
  81
  82             runner = mock.MagicMock()
  83             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84             runner.ignore_docker_for_reuse = False
  85             runner.intermediate_output_ttl = 0
  86             runner.secret_store = cwltool.secrets.SecretStore()
  87
  88             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  89             runner.api.collections().get().execute.return_value = {
  90                 "portable_data_hash": "99999999999999999999999999999993+99"}
  91
  92             tool = cmap({
  93                 "inputs": [],
  94                 "outputs": [],
  95                 "baseCommand": "ls",
  96                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  97                 "id": "#",
  98                 "class": "CommandLineTool"
  99             })
 100
 101             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 102
 103             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 104             arvtool.formatgraph = None
 105
 106             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 107                 j.run(runtimeContext)
 108                 runner.api.container_requests().create.assert_called_with(
 109                     body=JsonDiffMatcher({
 110                         'environment': {
 111                             'HOME': '/var/spool/cwl',
 112                             'TMPDIR': '/tmp'
 113                         },
 114                         'name': 'test_run_'+str(enable_reuse),
 115                         'runtime_constraints': {
 116                             'vcpus': 1,
 117                             'ram': 1073741824
 118                         },
 119                         'use_existing': enable_reuse,
 120                         'priority': 500,
 121                         'mounts': {
 122                             '/tmp': {'kind': 'tmp',
 123                                      "capacity": 1073741824
 124                                  },
 125                             '/var/spool/cwl': {'kind': 'tmp',
 126                                                "capacity": 1073741824 }
 127                         },
 128                         'state': 'Committed',
 129                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 130                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 131                         'output_path': '/var/spool/cwl',
 132                         'output_ttl': 0,
 133                         'container_image': 'arvados/jobs',
 134                         'command': ['ls', '/var/spool/cwl'],
 135                         'cwd': '/var/spool/cwl',
 136                         'scheduling_parameters': {},
 137                         'properties': {},
 138                         'secret_mounts': {}
 139                     }))
 140
 141     # The test passes some fields in builder.resources
 142     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 143     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 144     def test_resource_requirements(self, keepdocker):
 145         arv_docker_clear_cache()
 146         runner = mock.MagicMock()
 147         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 148         runner.ignore_docker_for_reuse = False
 149         runner.intermediate_output_ttl = 3600
 150         runner.secret_store = cwltool.secrets.SecretStore()
 151
 152         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 153         runner.api.collections().get().execute.return_value = {
 154             "portable_data_hash": "99999999999999999999999999999993+99"}
 155
 156         tool = cmap({
 157             "inputs": [],
 158             "outputs": [],
 159             "hints": [{
 160                 "class": "ResourceRequirement",
 161                 "coresMin": 3,
 162                 "ramMin": 3000,
 163                 "tmpdirMin": 4000,
 164                 "outdirMin": 5000
 165             }, {
 166                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 167                 "keep_cache": 512
 168             }, {
 169                 "class": "http://arvados.org/cwl#APIRequirement",
 170             }, {
 171                 "class": "http://arvados.org/cwl#PartitionRequirement",
 172                 "partition": "blurb"
 173             }, {
 174                 "class": "http://arvados.org/cwl#IntermediateOutput",
 175                 "outputTTL": 7200
 176             }, {
 177                 "class": "http://arvados.org/cwl#ReuseRequirement",
 178                 "enableReuse": False
 179             }],
 180             "baseCommand": "ls",
 181             "id": "#",
 182             "class": "CommandLineTool"
 183         })
 184
 185         loadingContext, runtimeContext = self.helper(runner)
 186         runtimeContext.name = "test_resource_requirements"
 187
 188         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 189         arvtool.formatgraph = None
 190         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 191             j.run(runtimeContext)
 192
 193         call_args, call_kwargs = runner.api.container_requests().create.call_args
 194
 195         call_body_expected = {
 196             'environment': {
 197                 'HOME': '/var/spool/cwl',
 198                 'TMPDIR': '/tmp'
 199             },
 200             'name': 'test_resource_requirements',
 201             'runtime_constraints': {
 202                 'vcpus': 3,
 203                 'ram': 3145728000,
 204                 'keep_cache_ram': 536870912,
 205                 'API': True
 206             },
 207             'use_existing': False,
 208             'priority': 500,
 209             'mounts': {
 210                 '/tmp': {'kind': 'tmp',
 211                          "capacity": 4194304000 },
 212                 '/var/spool/cwl': {'kind': 'tmp',
 213                                    "capacity": 5242880000 }
 214             },
 215             'state': 'Committed',
 216             'output_name': 'Output for step test_resource_requirements',
 217             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 218             'output_path': '/var/spool/cwl',
 219             'output_ttl': 7200,
 220             'container_image': 'arvados/jobs',
 221             'command': ['ls'],
 222             'cwd': '/var/spool/cwl',
 223             'scheduling_parameters': {
 224                 'partitions': ['blurb']
 225             },
 226             'properties': {},
 227             'secret_mounts': {}
 228         }
 229
 230         call_body = call_kwargs.get('body', None)
 231         self.assertNotEqual(None, call_body)
 232         for key in call_body:
 233             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 234
 235
 236     # The test passes some fields in builder.resources
 237     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 238     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 239     @mock.patch("arvados.collection.Collection")
 240     def test_initial_work_dir(self, collection_mock, keepdocker):
 241         arv_docker_clear_cache()
 242         runner = mock.MagicMock()
 243         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 244         runner.ignore_docker_for_reuse = False
 245         runner.intermediate_output_ttl = 0
 246         runner.secret_store = cwltool.secrets.SecretStore()
 247
 248         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 249         runner.api.collections().get().execute.return_value = {
 250             "portable_data_hash": "99999999999999999999999999999993+99"}
 251
 252         sourcemock = mock.MagicMock()
 253         def get_collection_mock(p):
 254             if "/" in p:
 255                 return (sourcemock, p.split("/", 1)[1])
 256             else:
 257                 return (sourcemock, "")
 258         runner.fs_access.get_collection.side_effect = get_collection_mock
 259
 260         vwdmock = mock.MagicMock()
 261         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 262
 263         tool = cmap({
 264             "inputs": [],
 265             "outputs": [],
 266             "hints": [{
 267                 "class": "InitialWorkDirRequirement",
 268                 "listing": [{
 269                     "class": "File",
 270                     "basename": "foo",
 271                     "location": "keep:99999999999999999999999999999995+99/bar"
 272                 },
 273                 {
 274                     "class": "Directory",
 275                     "basename": "foo2",
 276                     "location": "keep:99999999999999999999999999999995+99"
 277                 },
 278                 {
 279                     "class": "File",
 280                     "basename": "filename",
 281                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 282                 },
 283                 {
 284                     "class": "Directory",
 285                     "basename": "subdir",
 286                     "location": "keep:99999999999999999999999999999995+99/subdir"
 287                 }                        ]
 288             }],
 289             "baseCommand": "ls",
 290             "id": "#",
 291             "class": "CommandLineTool"
 292         })
 293
 294         loadingContext, runtimeContext = self.helper(runner)
 295         runtimeContext.name = "test_initial_work_dir"
 296
 297         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 298         arvtool.formatgraph = None
 299         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 300             j.run(runtimeContext)
 301
 302         call_args, call_kwargs = runner.api.container_requests().create.call_args
 303
 304         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 305         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 306         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 307         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 308
 309         call_body_expected = {
 310             'environment': {
 311                 'HOME': '/var/spool/cwl',
 312                 'TMPDIR': '/tmp'
 313             },
 314             'name': 'test_initial_work_dir',
 315             'runtime_constraints': {
 316                 'vcpus': 1,
 317                 'ram': 1073741824
 318             },
 319             'use_existing': True,
 320             'priority': 500,
 321             'mounts': {
 322                 '/tmp': {'kind': 'tmp',
 323                          "capacity": 1073741824 },
 324                 '/var/spool/cwl': {'kind': 'tmp',
 325                                    "capacity": 1073741824 },
 326                 '/var/spool/cwl/foo': {
 327                     'kind': 'collection',
 328                     'path': 'foo',
 329                     'portable_data_hash': '99999999999999999999999999999996+99'
 330                 },
 331                 '/var/spool/cwl/foo2': {
 332                     'kind': 'collection',
 333                     'path': 'foo2',
 334                     'portable_data_hash': '99999999999999999999999999999996+99'
 335                 },
 336                 '/var/spool/cwl/filename': {
 337                     'kind': 'collection',
 338                     'path': 'filename',
 339                     'portable_data_hash': '99999999999999999999999999999996+99'
 340                 },
 341                 '/var/spool/cwl/subdir': {
 342                     'kind': 'collection',
 343                     'path': 'subdir',
 344                     'portable_data_hash': '99999999999999999999999999999996+99'
 345                 }
 346             },
 347             'state': 'Committed',
 348             'output_name': 'Output for step test_initial_work_dir',
 349             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 350             'output_path': '/var/spool/cwl',
 351             'output_ttl': 0,
 352             'container_image': 'arvados/jobs',
 353             'command': ['ls'],
 354             'cwd': '/var/spool/cwl',
 355             'scheduling_parameters': {
 356             },
 357             'properties': {},
 358             'secret_mounts': {}
 359         }
 360
 361         call_body = call_kwargs.get('body', None)
 362         self.assertNotEqual(None, call_body)
 363         for key in call_body:
 364             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 365
 366
 367     # Test redirecting stdin/stdout/stderr
 368     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 369     def test_redirects(self, keepdocker):
 370         arv_docker_clear_cache()
 371
 372         runner = mock.MagicMock()
 373         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 374         runner.ignore_docker_for_reuse = False
 375         runner.intermediate_output_ttl = 0
 376         runner.secret_store = cwltool.secrets.SecretStore()
 377
 378         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 379         runner.api.collections().get().execute.return_value = {
 380             "portable_data_hash": "99999999999999999999999999999993+99"}
 381
 382         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 383
 384         tool = cmap({
 385             "inputs": [],
 386             "outputs": [],
 387             "baseCommand": "ls",
 388             "stdout": "stdout.txt",
 389             "stderr": "stderr.txt",
 390             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 391             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 392             "id": "#",
 393             "class": "CommandLineTool"
 394         })
 395
 396         loadingContext, runtimeContext = self.helper(runner)
 397         runtimeContext.name = "test_run_redirect"
 398
 399         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 400         arvtool.formatgraph = None
 401         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 402             j.run(runtimeContext)
 403             runner.api.container_requests().create.assert_called_with(
 404                 body=JsonDiffMatcher({
 405                     'environment': {
 406                         'HOME': '/var/spool/cwl',
 407                         'TMPDIR': '/tmp'
 408                     },
 409                     'name': 'test_run_redirect',
 410                     'runtime_constraints': {
 411                         'vcpus': 1,
 412                         'ram': 1073741824
 413                     },
 414                     'use_existing': True,
 415                     'priority': 500,
 416                     'mounts': {
 417                         '/tmp': {'kind': 'tmp',
 418                                  "capacity": 1073741824 },
 419                         '/var/spool/cwl': {'kind': 'tmp',
 420                                            "capacity": 1073741824 },
 421                         "stderr": {
 422                             "kind": "file",
 423                             "path": "/var/spool/cwl/stderr.txt"
 424                         },
 425                         "stdin": {
 426                             "kind": "collection",
 427                             "path": "file.txt",
 428                             "portable_data_hash": "99999999999999999999999999999996+99"
 429                         },
 430                         "stdout": {
 431                             "kind": "file",
 432                             "path": "/var/spool/cwl/stdout.txt"
 433                         },
 434                     },
 435                     'state': 'Committed',
 436                     "output_name": "Output for step test_run_redirect",
 437                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 438                     'output_path': '/var/spool/cwl',
 439                     'output_ttl': 0,
 440                     'container_image': 'arvados/jobs',
 441                     'command': ['ls', '/var/spool/cwl'],
 442                     'cwd': '/var/spool/cwl',
 443                     'scheduling_parameters': {},
 444                     'properties': {},
 445                     'secret_mounts': {}
 446                 }))
 447
 448     @mock.patch("arvados.collection.Collection")
 449     def test_done(self, col):
 450         api = mock.MagicMock()
 451
 452         runner = mock.MagicMock()
 453         runner.api = api
 454         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 455         runner.num_retries = 0
 456         runner.ignore_docker_for_reuse = False
 457         runner.intermediate_output_ttl = 0
 458         runner.secret_store = cwltool.secrets.SecretStore()
 459
 460         runner.api.containers().get().execute.return_value = {"state":"Complete",
 461                                                               "output": "abc+123",
 462                                                               "exit_code": 0}
 463
 464         col().open.return_value = []
 465
 466         arvjob = arvados_cwl.ArvadosContainer(runner,
 467                                               mock.MagicMock(),
 468                                               {},
 469                                               None,
 470                                               [],
 471                                               [],
 472                                               "testjob")
 473         arvjob.output_callback = mock.MagicMock()
 474         arvjob.collect_outputs = mock.MagicMock()
 475         arvjob.successCodes = [0]
 476         arvjob.outdir = "/var/spool/cwl"
 477         arvjob.output_ttl = 3600
 478
 479         arvjob.collect_outputs.return_value = {"out": "stuff"}
 480
 481         arvjob.done({
 482             "state": "Final",
 483             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 484             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 485             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 486             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 487             "modified_at": "2017-05-26T12:01:22Z"
 488         })
 489
 490         self.assertFalse(api.collections().create.called)
 491
 492         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 493         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 494         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 495
 496     # The test passes no builder.resources
 497     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 498     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 499     def test_mounts(self, keepdocker):
 500         arv_docker_clear_cache()
 501
 502         runner = mock.MagicMock()
 503         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 504         runner.ignore_docker_for_reuse = False
 505         runner.intermediate_output_ttl = 0
 506         runner.secret_store = cwltool.secrets.SecretStore()
 507
 508         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 509         runner.api.collections().get().execute.return_value = {
 510             "portable_data_hash": "99999999999999999999999999999994+99",
 511             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 512
 513         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 514
 515         tool = cmap({
 516             "inputs": [
 517                 {"id": "p1",
 518                  "type": "Directory"}
 519             ],
 520             "outputs": [],
 521             "baseCommand": "ls",
 522             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 523             "id": "#",
 524             "class": "CommandLineTool"
 525         })
 526
 527         loadingContext, runtimeContext = self.helper(runner)
 528         runtimeContext.name = "test_run_mounts"
 529
 530         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 531         arvtool.formatgraph = None
 532         job_order = {
 533             "p1": {
 534                 "class": "Directory",
 535                 "location": "keep:99999999999999999999999999999994+44",
 536                 "listing": [
 537                     {
 538                         "class": "File",
 539                         "location": "keep:99999999999999999999999999999994+44/file1",
 540                     },
 541                     {
 542                         "class": "File",
 543                         "location": "keep:99999999999999999999999999999994+44/file2",
 544                     }
 545                 ]
 546             }
 547         }
 548         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 549             j.run(runtimeContext)
 550             runner.api.container_requests().create.assert_called_with(
 551                 body=JsonDiffMatcher({
 552                     'environment': {
 553                         'HOME': '/var/spool/cwl',
 554                         'TMPDIR': '/tmp'
 555                     },
 556                     'name': 'test_run_mounts',
 557                     'runtime_constraints': {
 558                         'vcpus': 1,
 559                         'ram': 1073741824
 560                     },
 561                     'use_existing': True,
 562                     'priority': 500,
 563                     'mounts': {
 564                         "/keep/99999999999999999999999999999994+44": {
 565                             "kind": "collection",
 566                             "portable_data_hash": "99999999999999999999999999999994+44"
 567                         },
 568                         '/tmp': {'kind': 'tmp',
 569                                  "capacity": 1073741824 },
 570                         '/var/spool/cwl': {'kind': 'tmp',
 571                                            "capacity": 1073741824 }
 572                     },
 573                     'state': 'Committed',
 574                     'output_name': 'Output for step test_run_mounts',
 575                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 576                     'output_path': '/var/spool/cwl',
 577                     'output_ttl': 0,
 578                     'container_image': 'arvados/jobs',
 579                     'command': ['ls', '/var/spool/cwl'],
 580                     'cwd': '/var/spool/cwl',
 581                     'scheduling_parameters': {},
 582                     'properties': {},
 583                     'secret_mounts': {}
 584                 }))
 585
 586     # The test passes no builder.resources
 587     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 588     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 589     def test_secrets(self, keepdocker):
 590         arv_docker_clear_cache()
 591
 592         runner = mock.MagicMock()
 593         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 594         runner.ignore_docker_for_reuse = False
 595         runner.intermediate_output_ttl = 0
 596         runner.secret_store = cwltool.secrets.SecretStore()
 597
 598         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 599         runner.api.collections().get().execute.return_value = {
 600             "portable_data_hash": "99999999999999999999999999999993+99"}
 601
 602         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 603
 604         tool = cmap({"arguments": ["md5sum", "example.conf"],
 605                      "class": "CommandLineTool",
 606                      "hints": [
 607                          {
 608                              "class": "http://commonwl.org/cwltool#Secrets",
 609                              "secrets": [
 610                                  "#secret_job.cwl/pw"
 611                              ]
 612                          }
 613                      ],
 614                      "id": "#secret_job.cwl",
 615                      "inputs": [
 616                          {
 617                              "id": "#secret_job.cwl/pw",
 618                              "type": "string"
 619                          }
 620                      ],
 621                      "outputs": [
 622                      ],
 623                      "requirements": [
 624                          {
 625                              "class": "InitialWorkDirRequirement",
 626                              "listing": [
 627                                  {
 628                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 629                                      "entryname": "example.conf"
 630                                  }
 631                              ]
 632                          }
 633                      ]})
 634
 635         loadingContext, runtimeContext = self.helper(runner)
 636         runtimeContext.name = "test_secrets"
 637
 638         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 639         arvtool.formatgraph = None
 640
 641         job_order = {"pw": "blorp"}
 642         runner.secret_store.store(["pw"], job_order)
 643
 644         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 645             j.run(runtimeContext)
 646             runner.api.container_requests().create.assert_called_with(
 647                 body=JsonDiffMatcher({
 648                     'environment': {
 649                         'HOME': '/var/spool/cwl',
 650                         'TMPDIR': '/tmp'
 651                     },
 652                     'name': 'test_secrets',
 653                     'runtime_constraints': {
 654                         'vcpus': 1,
 655                         'ram': 1073741824
 656                     },
 657                     'use_existing': True,
 658                     'priority': 500,
 659                     'mounts': {
 660                         '/tmp': {'kind': 'tmp',
 661                                  "capacity": 1073741824
 662                              },
 663                         '/var/spool/cwl': {'kind': 'tmp',
 664                                            "capacity": 1073741824 }
 665                     },
 666                     'state': 'Committed',
 667                     'output_name': 'Output for step test_secrets',
 668                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 669                     'output_path': '/var/spool/cwl',
 670                     'output_ttl': 0,
 671                     'container_image': 'arvados/jobs',
 672                     'command': ['md5sum', 'example.conf'],
 673                     'cwd': '/var/spool/cwl',
 674                     'scheduling_parameters': {},
 675                     'properties': {},
 676                     "secret_mounts": {
 677                         "/var/spool/cwl/example.conf": {
 678                             "content": "username: user\npassword: blorp\n",
 679                             "kind": "text"
 680                         }
 681                     }
 682                 }))
 683
 684     # The test passes no builder.resources
 685     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 686     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 687     def test_timelimit(self, keepdocker):
 688         arv_docker_clear_cache()
 689
 690         runner = mock.MagicMock()
 691         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 692         runner.ignore_docker_for_reuse = False
 693         runner.intermediate_output_ttl = 0
 694         runner.secret_store = cwltool.secrets.SecretStore()
 695
 696         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 697         runner.api.collections().get().execute.return_value = {
 698             "portable_data_hash": "99999999999999999999999999999993+99"}
 699
 700         tool = cmap({
 701             "inputs": [],
 702             "outputs": [],
 703             "baseCommand": "ls",
 704             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 705             "id": "#",
 706             "class": "CommandLineTool",
 707             "hints": [
 708                 {
 709                     "class": "http://commonwl.org/cwltool#TimeLimit",
 710                     "timelimit": 42
 711                 }
 712             ]
 713         })
 714
 715         loadingContext, runtimeContext = self.helper(runner)
 716         runtimeContext.name = "test_timelimit"
 717
 718         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 719         arvtool.formatgraph = None
 720
 721         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 722             j.run(runtimeContext)
 723
 724         _, kwargs = runner.api.container_requests().create.call_args
 725         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))