sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 import arvados_cwl.context
   7 from arvados_cwl.arvdocker import arv_docker_clear_cache
   8 import logging
   9 import mock
  10 import unittest
  11 import os
  12 import functools
  13 import cwltool.process
  14 import cwltool.secrets
  15 from schema_salad.ref_resolver import Loader
  16 from schema_salad.sourceline import cmap
  17
  18 from .matcher import JsonDiffMatcher
  19
  20 if not os.getenv('ARVADOS_DEBUG'):
  21     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  22     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  23
  24
  25 class TestContainer(unittest.TestCase):
  26
  27     def helper(self, runner, enable_reuse=True):
  28         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  29
  30         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  31                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  32         loadingContext = arvados_cwl.context.ArvLoadingContext(
  33             {"avsc_names": avsc_names,
  34              "basedir": "",
  35              "make_fs_access": make_fs_access,
  36              "loader": Loader({}),
  37              "metadata": {"cwlVersion": "v1.0"}})
  38         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  39             {"work_api": "containers",
  40              "basedir": "",
  41              "name": "test_run_"+str(enable_reuse),
  42              "make_fs_access": make_fs_access,
  43              "tmpdir": "/tmp",
  44              "enable_reuse": enable_reuse,
  45              "priority": 500})
  46
  47         return loadingContext, runtimeContext
  48
  49     # The test passes no builder.resources
  50     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  51     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  52     def test_run(self, keepdocker):
  53         for enable_reuse in (True, False):
  54             arv_docker_clear_cache()
  55
  56             runner = mock.MagicMock()
  57             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  58             runner.ignore_docker_for_reuse = False
  59             runner.intermediate_output_ttl = 0
  60             runner.secret_store = cwltool.secrets.SecretStore()
  61
  62             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  63             runner.api.collections().get().execute.return_value = {
  64                 "portable_data_hash": "99999999999999999999999999999993+99"}
  65
  66             tool = cmap({
  67                 "inputs": [],
  68                 "outputs": [],
  69                 "baseCommand": "ls",
  70                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  71                 "id": "#",
  72                 "class": "CommandLineTool"
  73             })
  74
  75             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
  76
  77             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
  78             arvtool.formatgraph = None
  79
  80             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
  81                 j.run(runtimeContext)
  82                 runner.api.container_requests().create.assert_called_with(
  83                     body=JsonDiffMatcher({
  84                         'environment': {
  85                             'HOME': '/var/spool/cwl',
  86                             'TMPDIR': '/tmp'
  87                         },
  88                         'name': 'test_run_'+str(enable_reuse),
  89                         'runtime_constraints': {
  90                             'vcpus': 1,
  91                             'ram': 1073741824
  92                         },
  93                         'use_existing': enable_reuse,
  94                         'priority': 500,
  95                         'mounts': {
  96                             '/tmp': {'kind': 'tmp',
  97                                      "capacity": 1073741824
  98                                  },
  99                             '/var/spool/cwl': {'kind': 'tmp',
 100                                                "capacity": 1073741824 }
 101                         },
 102                         'state': 'Committed',
 103                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 104                         'output_path': '/var/spool/cwl',
 105                         'output_ttl': 0,
 106                         'container_image': 'arvados/jobs',
 107                         'command': ['ls', '/var/spool/cwl'],
 108                         'cwd': '/var/spool/cwl',
 109                         'scheduling_parameters': {},
 110                         'properties': {},
 111                         'secret_mounts': {}
 112                     }))
 113
 114     # The test passes some fields in builder.resources
 115     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 116     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 117     def test_resource_requirements(self, keepdocker):
 118         arv_docker_clear_cache()
 119         runner = mock.MagicMock()
 120         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 121         runner.ignore_docker_for_reuse = False
 122         runner.intermediate_output_ttl = 3600
 123         runner.secret_store = cwltool.secrets.SecretStore()
 124
 125         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 126         runner.api.collections().get().execute.return_value = {
 127             "portable_data_hash": "99999999999999999999999999999993+99"}
 128
 129         tool = cmap({
 130             "inputs": [],
 131             "outputs": [],
 132             "hints": [{
 133                 "class": "ResourceRequirement",
 134                 "coresMin": 3,
 135                 "ramMin": 3000,
 136                 "tmpdirMin": 4000,
 137                 "outdirMin": 5000
 138             }, {
 139                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 140                 "keep_cache": 512
 141             }, {
 142                 "class": "http://arvados.org/cwl#APIRequirement",
 143             }, {
 144                 "class": "http://arvados.org/cwl#PartitionRequirement",
 145                 "partition": "blurb"
 146             }, {
 147                 "class": "http://arvados.org/cwl#IntermediateOutput",
 148                 "outputTTL": 7200
 149             }, {
 150                 "class": "http://arvados.org/cwl#ReuseRequirement",
 151                 "enableReuse": False
 152             }],
 153             "baseCommand": "ls",
 154             "id": "#",
 155             "class": "CommandLineTool"
 156         })
 157
 158         loadingContext, runtimeContext = self.helper(runner)
 159         runtimeContext.name = "test_resource_requirements"
 160
 161         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 162         arvtool.formatgraph = None
 163         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 164             j.run(runtimeContext)
 165
 166         call_args, call_kwargs = runner.api.container_requests().create.call_args
 167
 168         call_body_expected = {
 169             'environment': {
 170                 'HOME': '/var/spool/cwl',
 171                 'TMPDIR': '/tmp'
 172             },
 173             'name': 'test_resource_requirements',
 174             'runtime_constraints': {
 175                 'vcpus': 3,
 176                 'ram': 3145728000,
 177                 'keep_cache_ram': 536870912,
 178                 'API': True
 179             },
 180             'use_existing': False,
 181             'priority': 500,
 182             'mounts': {
 183                 '/tmp': {'kind': 'tmp',
 184                          "capacity": 4194304000 },
 185                 '/var/spool/cwl': {'kind': 'tmp',
 186                                    "capacity": 5242880000 }
 187             },
 188             'state': 'Committed',
 189             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 190             'output_path': '/var/spool/cwl',
 191             'output_ttl': 7200,
 192             'container_image': 'arvados/jobs',
 193             'command': ['ls'],
 194             'cwd': '/var/spool/cwl',
 195             'scheduling_parameters': {
 196                 'partitions': ['blurb']
 197             },
 198             'properties': {},
 199             'secret_mounts': {}
 200         }
 201
 202         call_body = call_kwargs.get('body', None)
 203         self.assertNotEqual(None, call_body)
 204         for key in call_body:
 205             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 206
 207
 208     # The test passes some fields in builder.resources
 209     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 210     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 211     @mock.patch("arvados.collection.Collection")
 212     def test_initial_work_dir(self, collection_mock, keepdocker):
 213         arv_docker_clear_cache()
 214         runner = mock.MagicMock()
 215         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 216         runner.ignore_docker_for_reuse = False
 217         runner.intermediate_output_ttl = 0
 218         runner.secret_store = cwltool.secrets.SecretStore()
 219
 220         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 221         runner.api.collections().get().execute.return_value = {
 222             "portable_data_hash": "99999999999999999999999999999993+99"}
 223
 224         sourcemock = mock.MagicMock()
 225         def get_collection_mock(p):
 226             if "/" in p:
 227                 return (sourcemock, p.split("/", 1)[1])
 228             else:
 229                 return (sourcemock, "")
 230         runner.fs_access.get_collection.side_effect = get_collection_mock
 231
 232         vwdmock = mock.MagicMock()
 233         collection_mock.return_value = vwdmock
 234         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 235
 236         tool = cmap({
 237             "inputs": [],
 238             "outputs": [],
 239             "hints": [{
 240                 "class": "InitialWorkDirRequirement",
 241                 "listing": [{
 242                     "class": "File",
 243                     "basename": "foo",
 244                     "location": "keep:99999999999999999999999999999995+99/bar"
 245                 },
 246                 {
 247                     "class": "Directory",
 248                     "basename": "foo2",
 249                     "location": "keep:99999999999999999999999999999995+99"
 250                 },
 251                 {
 252                     "class": "File",
 253                     "basename": "filename",
 254                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 255                 },
 256                 {
 257                     "class": "Directory",
 258                     "basename": "subdir",
 259                     "location": "keep:99999999999999999999999999999995+99/subdir"
 260                 }                        ]
 261             }],
 262             "baseCommand": "ls",
 263             "id": "#",
 264             "class": "CommandLineTool"
 265         })
 266
 267         loadingContext, runtimeContext = self.helper(runner)
 268         runtimeContext.name = "test_initial_work_dir"
 269
 270         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 271         arvtool.formatgraph = None
 272         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 273             j.run(runtimeContext)
 274
 275         call_args, call_kwargs = runner.api.container_requests().create.call_args
 276
 277         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 278         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 279         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 280         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 281
 282         call_body_expected = {
 283             'environment': {
 284                 'HOME': '/var/spool/cwl',
 285                 'TMPDIR': '/tmp'
 286             },
 287             'name': 'test_initial_work_dir',
 288             'runtime_constraints': {
 289                 'vcpus': 1,
 290                 'ram': 1073741824
 291             },
 292             'use_existing': True,
 293             'priority': 500,
 294             'mounts': {
 295                 '/tmp': {'kind': 'tmp',
 296                          "capacity": 1073741824 },
 297                 '/var/spool/cwl': {'kind': 'tmp',
 298                                    "capacity": 1073741824 },
 299                 '/var/spool/cwl/foo': {
 300                     'kind': 'collection',
 301                     'path': 'foo',
 302                     'portable_data_hash': '99999999999999999999999999999996+99'
 303                 },
 304                 '/var/spool/cwl/foo2': {
 305                     'kind': 'collection',
 306                     'path': 'foo2',
 307                     'portable_data_hash': '99999999999999999999999999999996+99'
 308                 },
 309                 '/var/spool/cwl/filename': {
 310                     'kind': 'collection',
 311                     'path': 'filename',
 312                     'portable_data_hash': '99999999999999999999999999999996+99'
 313                 },
 314                 '/var/spool/cwl/subdir': {
 315                     'kind': 'collection',
 316                     'path': 'subdir',
 317                     'portable_data_hash': '99999999999999999999999999999996+99'
 318                 }
 319             },
 320             'state': 'Committed',
 321             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 322             'output_path': '/var/spool/cwl',
 323             'output_ttl': 0,
 324             'container_image': 'arvados/jobs',
 325             'command': ['ls'],
 326             'cwd': '/var/spool/cwl',
 327             'scheduling_parameters': {
 328             },
 329             'properties': {},
 330             'secret_mounts': {}
 331         }
 332
 333         call_body = call_kwargs.get('body', None)
 334         self.assertNotEqual(None, call_body)
 335         for key in call_body:
 336             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 337
 338
 339     # Test redirecting stdin/stdout/stderr
 340     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 341     def test_redirects(self, keepdocker):
 342         arv_docker_clear_cache()
 343
 344         runner = mock.MagicMock()
 345         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 346         runner.ignore_docker_for_reuse = False
 347         runner.intermediate_output_ttl = 0
 348         runner.secret_store = cwltool.secrets.SecretStore()
 349
 350         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 351         runner.api.collections().get().execute.return_value = {
 352             "portable_data_hash": "99999999999999999999999999999993+99"}
 353
 354         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 355
 356         tool = cmap({
 357             "inputs": [],
 358             "outputs": [],
 359             "baseCommand": "ls",
 360             "stdout": "stdout.txt",
 361             "stderr": "stderr.txt",
 362             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 363             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 364             "id": "#",
 365             "class": "CommandLineTool"
 366         })
 367
 368         loadingContext, runtimeContext = self.helper(runner)
 369         runtimeContext.name = "test_run_redirect"
 370
 371         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 372         arvtool.formatgraph = None
 373         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 374             j.run(runtimeContext)
 375             runner.api.container_requests().create.assert_called_with(
 376                 body=JsonDiffMatcher({
 377                     'environment': {
 378                         'HOME': '/var/spool/cwl',
 379                         'TMPDIR': '/tmp'
 380                     },
 381                     'name': 'test_run_redirect',
 382                     'runtime_constraints': {
 383                         'vcpus': 1,
 384                         'ram': 1073741824
 385                     },
 386                     'use_existing': True,
 387                     'priority': 500,
 388                     'mounts': {
 389                         '/tmp': {'kind': 'tmp',
 390                                  "capacity": 1073741824 },
 391                         '/var/spool/cwl': {'kind': 'tmp',
 392                                            "capacity": 1073741824 },
 393                         "stderr": {
 394                             "kind": "file",
 395                             "path": "/var/spool/cwl/stderr.txt"
 396                         },
 397                         "stdin": {
 398                             "kind": "collection",
 399                             "path": "file.txt",
 400                             "portable_data_hash": "99999999999999999999999999999996+99"
 401                         },
 402                         "stdout": {
 403                             "kind": "file",
 404                             "path": "/var/spool/cwl/stdout.txt"
 405                         },
 406                     },
 407                     'state': 'Committed',
 408                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 409                     'output_path': '/var/spool/cwl',
 410                     'output_ttl': 0,
 411                     'container_image': 'arvados/jobs',
 412                     'command': ['ls', '/var/spool/cwl'],
 413                     'cwd': '/var/spool/cwl',
 414                     'scheduling_parameters': {},
 415                     'properties': {},
 416                     'secret_mounts': {}
 417                 }))
 418
 419     @mock.patch("arvados.collection.Collection")
 420     def test_done(self, col):
 421         api = mock.MagicMock()
 422
 423         runner = mock.MagicMock()
 424         runner.api = api
 425         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 426         runner.num_retries = 0
 427         runner.ignore_docker_for_reuse = False
 428         runner.intermediate_output_ttl = 0
 429         runner.secret_store = cwltool.secrets.SecretStore()
 430
 431         runner.api.containers().get().execute.return_value = {"state":"Complete",
 432                                                               "output": "abc+123",
 433                                                               "exit_code": 0}
 434
 435         col().open.return_value = []
 436
 437         arvjob = arvados_cwl.ArvadosContainer(runner,
 438                                               mock.MagicMock(),
 439                                               {},
 440                                               None,
 441                                               [],
 442                                               [],
 443                                               "testjob")
 444         arvjob.output_callback = mock.MagicMock()
 445         arvjob.collect_outputs = mock.MagicMock()
 446         arvjob.successCodes = [0]
 447         arvjob.outdir = "/var/spool/cwl"
 448         arvjob.output_ttl = 3600
 449
 450         arvjob.collect_outputs.return_value = {"out": "stuff"}
 451
 452         arvjob.done({
 453             "state": "Final",
 454             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 455             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 456             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 457             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 458             "modified_at": "2017-05-26T12:01:22Z"
 459         })
 460
 461         self.assertFalse(api.collections().create.called)
 462
 463         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 464         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 465         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 466
 467     # The test passes no builder.resources
 468     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 469     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 470     def test_mounts(self, keepdocker):
 471         arv_docker_clear_cache()
 472
 473         runner = mock.MagicMock()
 474         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 475         runner.ignore_docker_for_reuse = False
 476         runner.intermediate_output_ttl = 0
 477         runner.secret_store = cwltool.secrets.SecretStore()
 478
 479         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 480         runner.api.collections().get().execute.return_value = {
 481             "portable_data_hash": "99999999999999999999999999999993+99"}
 482
 483         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 484
 485         tool = cmap({
 486             "inputs": [
 487                 {"id": "p1",
 488                  "type": "Directory"}
 489             ],
 490             "outputs": [],
 491             "baseCommand": "ls",
 492             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 493             "id": "#",
 494             "class": "CommandLineTool"
 495         })
 496
 497         loadingContext, runtimeContext = self.helper(runner)
 498         runtimeContext.name = "test_run_mounts"
 499
 500         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 501         arvtool.formatgraph = None
 502         job_order = {
 503             "p1": {
 504                 "class": "Directory",
 505                 "location": "keep:99999999999999999999999999999994+44",
 506                 "listing": [
 507                     {
 508                         "class": "File",
 509                         "location": "keep:99999999999999999999999999999994+44/file1",
 510                     },
 511                     {
 512                         "class": "File",
 513                         "location": "keep:99999999999999999999999999999994+44/file2",
 514                     }
 515                 ]
 516             }
 517         }
 518         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 519             j.run(runtimeContext)
 520             runner.api.container_requests().create.assert_called_with(
 521                 body=JsonDiffMatcher({
 522                     'environment': {
 523                         'HOME': '/var/spool/cwl',
 524                         'TMPDIR': '/tmp'
 525                     },
 526                     'name': 'test_run_mounts',
 527                     'runtime_constraints': {
 528                         'vcpus': 1,
 529                         'ram': 1073741824
 530                     },
 531                     'use_existing': True,
 532                     'priority': 500,
 533                     'mounts': {
 534                         "/keep/99999999999999999999999999999994+44": {
 535                             "kind": "collection",
 536                             "portable_data_hash": "99999999999999999999999999999994+44"
 537                         },
 538                         '/tmp': {'kind': 'tmp',
 539                                  "capacity": 1073741824 },
 540                         '/var/spool/cwl': {'kind': 'tmp',
 541                                            "capacity": 1073741824 }
 542                     },
 543                     'state': 'Committed',
 544                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 545                     'output_path': '/var/spool/cwl',
 546                     'output_ttl': 0,
 547                     'container_image': 'arvados/jobs',
 548                     'command': ['ls', '/var/spool/cwl'],
 549                     'cwd': '/var/spool/cwl',
 550                     'scheduling_parameters': {},
 551                     'properties': {},
 552                     'secret_mounts': {}
 553                 }))
 554
 555     # The test passes no builder.resources
 556     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 557     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 558     def test_secrets(self, keepdocker):
 559         arv_docker_clear_cache()
 560
 561         runner = mock.MagicMock()
 562         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 563         runner.ignore_docker_for_reuse = False
 564         runner.intermediate_output_ttl = 0
 565         runner.secret_store = cwltool.secrets.SecretStore()
 566
 567         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 568         runner.api.collections().get().execute.return_value = {
 569             "portable_data_hash": "99999999999999999999999999999993+99"}
 570
 571         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 572
 573         tool = cmap({"arguments": ["md5sum", "example.conf"],
 574                      "class": "CommandLineTool",
 575                      "hints": [
 576                          {
 577                              "class": "http://commonwl.org/cwltool#Secrets",
 578                              "secrets": [
 579                                  "#secret_job.cwl/pw"
 580                              ]
 581                          }
 582                      ],
 583                      "id": "#secret_job.cwl",
 584                      "inputs": [
 585                          {
 586                              "id": "#secret_job.cwl/pw",
 587                              "type": "string"
 588                          }
 589                      ],
 590                      "outputs": [
 591                      ],
 592                      "requirements": [
 593                          {
 594                              "class": "InitialWorkDirRequirement",
 595                              "listing": [
 596                                  {
 597                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 598                                      "entryname": "example.conf"
 599                                  }
 600                              ]
 601                          }
 602                      ]})
 603
 604         loadingContext, runtimeContext = self.helper(runner)
 605         runtimeContext.name = "test_secrets"
 606
 607         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 608         arvtool.formatgraph = None
 609
 610         job_order = {"pw": "blorp"}
 611         runner.secret_store.store(["pw"], job_order)
 612
 613         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 614             j.run(runtimeContext)
 615             runner.api.container_requests().create.assert_called_with(
 616                 body=JsonDiffMatcher({
 617                     'environment': {
 618                         'HOME': '/var/spool/cwl',
 619                         'TMPDIR': '/tmp'
 620                     },
 621                     'name': 'test_secrets',
 622                     'runtime_constraints': {
 623                         'vcpus': 1,
 624                         'ram': 1073741824
 625                     },
 626                     'use_existing': True,
 627                     'priority': 500,
 628                     'mounts': {
 629                         '/tmp': {'kind': 'tmp',
 630                                  "capacity": 1073741824
 631                              },
 632                         '/var/spool/cwl': {'kind': 'tmp',
 633                                            "capacity": 1073741824 }
 634                     },
 635                     'state': 'Committed',
 636                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 637                     'output_path': '/var/spool/cwl',
 638                     'output_ttl': 0,
 639                     'container_image': 'arvados/jobs',
 640                     'command': ['md5sum', 'example.conf'],
 641                     'cwd': '/var/spool/cwl',
 642                     'scheduling_parameters': {},
 643                     'properties': {},
 644                     "secret_mounts": {
 645                         "/var/spool/cwl/example.conf": {
 646                             "content": "username: user\npassword: blorp\n",
 647                             "kind": "text"
 648                         }
 649                     }
 650                 }))
 651
 652     # The test passes no builder.resources
 653     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 654     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 655     def test_timelimit(self, keepdocker):
 656         arv_docker_clear_cache()
 657
 658         runner = mock.MagicMock()
 659         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 660         runner.ignore_docker_for_reuse = False
 661         runner.intermediate_output_ttl = 0
 662         runner.secret_store = cwltool.secrets.SecretStore()
 663
 664         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 665         runner.api.collections().get().execute.return_value = {
 666             "portable_data_hash": "99999999999999999999999999999993+99"}
 667
 668         tool = cmap({
 669             "inputs": [],
 670             "outputs": [],
 671             "baseCommand": "ls",
 672             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 673             "id": "#",
 674             "class": "CommandLineTool",
 675             "hints": [
 676                 {
 677                     "class": "http://commonwl.org/cwltool#TimeLimit",
 678                     "timelimit": 42
 679                 }
 680             ]
 681         })
 682
 683         loadingContext, runtimeContext = self.helper(runner)
 684         runtimeContext.name = "test_timelimit"
 685
 686         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 687         arvtool.formatgraph = None
 688
 689         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 690             j.run(runtimeContext)
 691
 692         _, kwargs = runner.api.container_requests().create.call_args
 693         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))