sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import functools
  12 import cwltool.process
  13 import cwltool.secrets
  14 from schema_salad.ref_resolver import Loader
  15 from schema_salad.sourceline import cmap
  16
  17 from .matcher import JsonDiffMatcher
  18
  19 if not os.getenv('ARVADOS_DEBUG'):
  20     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  21     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  22
  23
  24 class TestContainer(unittest.TestCase):
  25
  26     # The test passes no builder.resources
  27     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  28     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  29     def test_run(self, keepdocker):
  30         for enable_reuse in (True, False):
  31             arv_docker_clear_cache()
  32
  33             runner = mock.MagicMock()
  34             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  35             runner.ignore_docker_for_reuse = False
  36             runner.intermediate_output_ttl = 0
  37             runner.secret_store = cwltool.secrets.SecretStore()
  38
  39             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  40             runner.api.collections().get().execute.return_value = {
  41                 "portable_data_hash": "99999999999999999999999999999993+99"}
  42
  43             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  44
  45             tool = cmap({
  46                 "inputs": [],
  47                 "outputs": [],
  48                 "baseCommand": "ls",
  49                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  50                 "id": "#",
  51                 "class": "CommandLineTool"
  52             })
  53             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  54                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  55             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  56                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}),
  57                                                      metadata={"cwlVersion": "v1.0"})
  58             arvtool.formatgraph = None
  59             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  60                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  61                 j.run(enable_reuse=enable_reuse, priority=500)
  62                 runner.api.container_requests().create.assert_called_with(
  63                     body=JsonDiffMatcher({
  64                         'environment': {
  65                             'HOME': '/var/spool/cwl',
  66                             'TMPDIR': '/tmp'
  67                         },
  68                         'name': 'test_run_'+str(enable_reuse),
  69                         'runtime_constraints': {
  70                             'vcpus': 1,
  71                             'ram': 1073741824
  72                         },
  73                         'use_existing': enable_reuse,
  74                         'priority': 500,
  75                         'mounts': {
  76                             '/tmp': {'kind': 'tmp',
  77                                      "capacity": 1073741824
  78                                  },
  79                             '/var/spool/cwl': {'kind': 'tmp',
  80                                                "capacity": 1073741824 }
  81                         },
  82                         'state': 'Committed',
  83                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  84                         'output_path': '/var/spool/cwl',
  85                         'output_ttl': 0,
  86                         'container_image': 'arvados/jobs',
  87                         'command': ['ls', '/var/spool/cwl'],
  88                         'cwd': '/var/spool/cwl',
  89                         'scheduling_parameters': {},
  90                         'properties': {},
  91                         'secret_mounts': {}
  92                     }))
  93
  94     # The test passes some fields in builder.resources
  95     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  96     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  97     def test_resource_requirements(self, keepdocker):
  98         arv_docker_clear_cache()
  99         runner = mock.MagicMock()
 100         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 101         runner.ignore_docker_for_reuse = False
 102         runner.intermediate_output_ttl = 3600
 103         runner.secret_store = cwltool.secrets.SecretStore()
 104
 105         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 106
 107         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 108         runner.api.collections().get().execute.return_value = {
 109             "portable_data_hash": "99999999999999999999999999999993+99"}
 110
 111         tool = cmap({
 112             "inputs": [],
 113             "outputs": [],
 114             "hints": [{
 115                 "class": "ResourceRequirement",
 116                 "coresMin": 3,
 117                 "ramMin": 3000,
 118                 "tmpdirMin": 4000,
 119                 "outdirMin": 5000
 120             }, {
 121                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 122                 "keep_cache": 512
 123             }, {
 124                 "class": "http://arvados.org/cwl#APIRequirement",
 125             }, {
 126                 "class": "http://arvados.org/cwl#PartitionRequirement",
 127                 "partition": "blurb"
 128             }, {
 129                 "class": "http://arvados.org/cwl#IntermediateOutput",
 130                 "outputTTL": 7200
 131             }, {
 132                 "class": "http://arvados.org/cwl#ReuseRequirement",
 133                 "enableReuse": False
 134             }],
 135             "baseCommand": "ls",
 136             "id": "#",
 137             "class": "CommandLineTool"
 138         })
 139         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 140                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 141         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 142                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 143                                                  loader=Loader({}), metadata={"cwlVersion": "v1.0"})
 144         arvtool.formatgraph = None
 145         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 146                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 147             j.run(enable_reuse=True, priority=500)
 148
 149         call_args, call_kwargs = runner.api.container_requests().create.call_args
 150
 151         call_body_expected = {
 152             'environment': {
 153                 'HOME': '/var/spool/cwl',
 154                 'TMPDIR': '/tmp'
 155             },
 156             'name': 'test_resource_requirements',
 157             'runtime_constraints': {
 158                 'vcpus': 3,
 159                 'ram': 3145728000,
 160                 'keep_cache_ram': 536870912,
 161                 'API': True
 162             },
 163             'use_existing': False,
 164             'priority': 500,
 165             'mounts': {
 166                 '/tmp': {'kind': 'tmp',
 167                          "capacity": 4194304000 },
 168                 '/var/spool/cwl': {'kind': 'tmp',
 169                                    "capacity": 5242880000 }
 170             },
 171             'state': 'Committed',
 172             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 173             'output_path': '/var/spool/cwl',
 174             'output_ttl': 7200,
 175             'container_image': 'arvados/jobs',
 176             'command': ['ls'],
 177             'cwd': '/var/spool/cwl',
 178             'scheduling_parameters': {
 179                 'partitions': ['blurb']
 180             },
 181             'properties': {},
 182             'secret_mounts': {}
 183         }
 184
 185         call_body = call_kwargs.get('body', None)
 186         self.assertNotEqual(None, call_body)
 187         for key in call_body:
 188             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 189
 190
 191     # The test passes some fields in builder.resources
 192     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 193     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 194     @mock.patch("arvados.collection.Collection")
 195     def test_initial_work_dir(self, collection_mock, keepdocker):
 196         arv_docker_clear_cache()
 197         runner = mock.MagicMock()
 198         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 199         runner.ignore_docker_for_reuse = False
 200         runner.intermediate_output_ttl = 0
 201         runner.secret_store = cwltool.secrets.SecretStore()
 202
 203         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 204
 205         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 206         runner.api.collections().get().execute.return_value = {
 207             "portable_data_hash": "99999999999999999999999999999993+99"}
 208
 209         sourcemock = mock.MagicMock()
 210         def get_collection_mock(p):
 211             if "/" in p:
 212                 return (sourcemock, p.split("/", 1)[1])
 213             else:
 214                 return (sourcemock, "")
 215         runner.fs_access.get_collection.side_effect = get_collection_mock
 216
 217         vwdmock = mock.MagicMock()
 218         collection_mock.return_value = vwdmock
 219         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 220
 221         tool = cmap({
 222             "inputs": [],
 223             "outputs": [],
 224             "hints": [{
 225                 "class": "InitialWorkDirRequirement",
 226                 "listing": [{
 227                     "class": "File",
 228                     "basename": "foo",
 229                     "location": "keep:99999999999999999999999999999995+99/bar"
 230                 },
 231                 {
 232                     "class": "Directory",
 233                     "basename": "foo2",
 234                     "location": "keep:99999999999999999999999999999995+99"
 235                 },
 236                 {
 237                     "class": "File",
 238                     "basename": "filename",
 239                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 240                 },
 241                 {
 242                     "class": "Directory",
 243                     "basename": "subdir",
 244                     "location": "keep:99999999999999999999999999999995+99/subdir"
 245                 }                        ]
 246             }],
 247             "baseCommand": "ls",
 248             "id": "#",
 249             "class": "CommandLineTool"
 250         })
 251         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 252                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 253         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 254                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 255                                                  loader=Loader({}), metadata={"cwlVersion": "v1.0"})
 256         arvtool.formatgraph = None
 257         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 258                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 259             j.run(priority=500)
 260
 261         call_args, call_kwargs = runner.api.container_requests().create.call_args
 262
 263         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 264         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 265         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 266         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 267
 268         call_body_expected = {
 269             'environment': {
 270                 'HOME': '/var/spool/cwl',
 271                 'TMPDIR': '/tmp'
 272             },
 273             'name': 'test_initial_work_dir',
 274             'runtime_constraints': {
 275                 'vcpus': 1,
 276                 'ram': 1073741824
 277             },
 278             'use_existing': True,
 279             'priority': 500,
 280             'mounts': {
 281                 '/tmp': {'kind': 'tmp',
 282                          "capacity": 1073741824 },
 283                 '/var/spool/cwl': {'kind': 'tmp',
 284                                    "capacity": 1073741824 },
 285                 '/var/spool/cwl/foo': {
 286                     'kind': 'collection',
 287                     'path': 'foo',
 288                     'portable_data_hash': '99999999999999999999999999999996+99'
 289                 },
 290                 '/var/spool/cwl/foo2': {
 291                     'kind': 'collection',
 292                     'path': 'foo2',
 293                     'portable_data_hash': '99999999999999999999999999999996+99'
 294                 },
 295                 '/var/spool/cwl/filename': {
 296                     'kind': 'collection',
 297                     'path': 'filename',
 298                     'portable_data_hash': '99999999999999999999999999999996+99'
 299                 },
 300                 '/var/spool/cwl/subdir': {
 301                     'kind': 'collection',
 302                     'path': 'subdir',
 303                     'portable_data_hash': '99999999999999999999999999999996+99'
 304                 }
 305             },
 306             'state': 'Committed',
 307             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 308             'output_path': '/var/spool/cwl',
 309             'output_ttl': 0,
 310             'container_image': 'arvados/jobs',
 311             'command': ['ls'],
 312             'cwd': '/var/spool/cwl',
 313             'scheduling_parameters': {
 314             },
 315             'properties': {},
 316             'secret_mounts': {}
 317         }
 318
 319         call_body = call_kwargs.get('body', None)
 320         self.assertNotEqual(None, call_body)
 321         for key in call_body:
 322             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 323
 324
 325     # Test redirecting stdin/stdout/stderr
 326     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 327     def test_redirects(self, keepdocker):
 328         arv_docker_clear_cache()
 329
 330         runner = mock.MagicMock()
 331         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 332         runner.ignore_docker_for_reuse = False
 333         runner.intermediate_output_ttl = 0
 334         runner.secret_store = cwltool.secrets.SecretStore()
 335
 336         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 337         runner.api.collections().get().execute.return_value = {
 338             "portable_data_hash": "99999999999999999999999999999993+99"}
 339
 340         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 341
 342         tool = cmap({
 343             "inputs": [],
 344             "outputs": [],
 345             "baseCommand": "ls",
 346             "stdout": "stdout.txt",
 347             "stderr": "stderr.txt",
 348             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 349             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 350             "id": "#",
 351             "class": "CommandLineTool"
 352         })
 353         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 354                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 355         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 356                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 357                                                  metadata={"cwlVersion": "v1.0"})
 358         arvtool.formatgraph = None
 359         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 360                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 361             j.run(priority=500)
 362             runner.api.container_requests().create.assert_called_with(
 363                 body=JsonDiffMatcher({
 364                     'environment': {
 365                         'HOME': '/var/spool/cwl',
 366                         'TMPDIR': '/tmp'
 367                     },
 368                     'name': 'test_run_redirect',
 369                     'runtime_constraints': {
 370                         'vcpus': 1,
 371                         'ram': 1073741824
 372                     },
 373                     'use_existing': True,
 374                     'priority': 500,
 375                     'mounts': {
 376                         '/tmp': {'kind': 'tmp',
 377                                  "capacity": 1073741824 },
 378                         '/var/spool/cwl': {'kind': 'tmp',
 379                                            "capacity": 1073741824 },
 380                         "stderr": {
 381                             "kind": "file",
 382                             "path": "/var/spool/cwl/stderr.txt"
 383                         },
 384                         "stdin": {
 385                             "kind": "collection",
 386                             "path": "file.txt",
 387                             "portable_data_hash": "99999999999999999999999999999996+99"
 388                         },
 389                         "stdout": {
 390                             "kind": "file",
 391                             "path": "/var/spool/cwl/stdout.txt"
 392                         },
 393                     },
 394                     'state': 'Committed',
 395                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 396                     'output_path': '/var/spool/cwl',
 397                     'output_ttl': 0,
 398                     'container_image': 'arvados/jobs',
 399                     'command': ['ls', '/var/spool/cwl'],
 400                     'cwd': '/var/spool/cwl',
 401                     'scheduling_parameters': {},
 402                     'properties': {},
 403                     'secret_mounts': {}
 404                 }))
 405
 406     @mock.patch("arvados.collection.Collection")
 407     def test_done(self, col):
 408         api = mock.MagicMock()
 409
 410         runner = mock.MagicMock()
 411         runner.api = api
 412         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 413         runner.num_retries = 0
 414         runner.ignore_docker_for_reuse = False
 415         runner.intermediate_output_ttl = 0
 416         runner.secret_store = cwltool.secrets.SecretStore()
 417
 418         runner.api.containers().get().execute.return_value = {"state":"Complete",
 419                                                               "output": "abc+123",
 420                                                               "exit_code": 0}
 421
 422         col().open.return_value = []
 423
 424         arvjob = arvados_cwl.ArvadosContainer(runner)
 425         arvjob.name = "testjob"
 426         arvjob.builder = mock.MagicMock()
 427         arvjob.output_callback = mock.MagicMock()
 428         arvjob.collect_outputs = mock.MagicMock()
 429         arvjob.successCodes = [0]
 430         arvjob.outdir = "/var/spool/cwl"
 431         arvjob.output_ttl = 3600
 432
 433         arvjob.collect_outputs.return_value = {"out": "stuff"}
 434
 435         arvjob.done({
 436             "state": "Final",
 437             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 438             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 439             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 440             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 441             "modified_at": "2017-05-26T12:01:22Z"
 442         })
 443
 444         self.assertFalse(api.collections().create.called)
 445
 446         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 447         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 448         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 449
 450     # The test passes no builder.resources
 451     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 452     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 453     def test_mounts(self, keepdocker):
 454         arv_docker_clear_cache()
 455
 456         runner = mock.MagicMock()
 457         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 458         runner.ignore_docker_for_reuse = False
 459         runner.intermediate_output_ttl = 0
 460         runner.secret_store = cwltool.secrets.SecretStore()
 461
 462         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 463         runner.api.collections().get().execute.return_value = {
 464             "portable_data_hash": "99999999999999999999999999999993+99"}
 465
 466         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 467
 468         tool = cmap({
 469             "inputs": [
 470                 {"id": "p1",
 471                  "type": "Directory"}
 472             ],
 473             "outputs": [],
 474             "baseCommand": "ls",
 475             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 476             "id": "#",
 477             "class": "CommandLineTool"
 478         })
 479         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 480                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 481         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 482                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 483                                                  metadata={"cwlVersion": "v1.0"})
 484         arvtool.formatgraph = None
 485         job_order = {
 486             "p1": {
 487                 "class": "Directory",
 488                 "location": "keep:99999999999999999999999999999994+44",
 489                 "listing": [
 490                     {
 491                         "class": "File",
 492                         "location": "keep:99999999999999999999999999999994+44/file1",
 493                     },
 494                     {
 495                         "class": "File",
 496                         "location": "keep:99999999999999999999999999999994+44/file2",
 497                     }
 498                 ]
 499             }
 500         }
 501         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 502                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 503             j.run(priority=500)
 504             runner.api.container_requests().create.assert_called_with(
 505                 body=JsonDiffMatcher({
 506                     'environment': {
 507                         'HOME': '/var/spool/cwl',
 508                         'TMPDIR': '/tmp'
 509                     },
 510                     'name': 'test_run_mounts',
 511                     'runtime_constraints': {
 512                         'vcpus': 1,
 513                         'ram': 1073741824
 514                     },
 515                     'use_existing': True,
 516                     'priority': 500,
 517                     'mounts': {
 518                         "/keep/99999999999999999999999999999994+44": {
 519                             "kind": "collection",
 520                             "portable_data_hash": "99999999999999999999999999999994+44"
 521                         },
 522                         '/tmp': {'kind': 'tmp',
 523                                  "capacity": 1073741824 },
 524                         '/var/spool/cwl': {'kind': 'tmp',
 525                                            "capacity": 1073741824 }
 526                     },
 527                     'state': 'Committed',
 528                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 529                     'output_path': '/var/spool/cwl',
 530                     'output_ttl': 0,
 531                     'container_image': 'arvados/jobs',
 532                     'command': ['ls', '/var/spool/cwl'],
 533                     'cwd': '/var/spool/cwl',
 534                     'scheduling_parameters': {},
 535                     'properties': {},
 536                     'secret_mounts': {}
 537                 }))
 538
 539     # The test passes no builder.resources
 540     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 541     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 542     def test_secrets(self, keepdocker):
 543         arv_docker_clear_cache()
 544
 545         runner = mock.MagicMock()
 546         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 547         runner.ignore_docker_for_reuse = False
 548         runner.intermediate_output_ttl = 0
 549         runner.secret_store = cwltool.secrets.SecretStore()
 550
 551         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 552         runner.api.collections().get().execute.return_value = {
 553             "portable_data_hash": "99999999999999999999999999999993+99"}
 554
 555         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 556
 557         tool = cmap({"arguments": ["md5sum", "example.conf"],
 558                      "class": "CommandLineTool",
 559                      "hints": [
 560                          {
 561                              "class": "http://commonwl.org/cwltool#Secrets",
 562                              "secrets": [
 563                                  "#secret_job.cwl/pw"
 564                              ]
 565                          }
 566                      ],
 567                      "id": "#secret_job.cwl",
 568                      "inputs": [
 569                          {
 570                              "id": "#secret_job.cwl/pw",
 571                              "type": "string"
 572                          }
 573                      ],
 574                      "outputs": [
 575                      ],
 576                      "requirements": [
 577                          {
 578                              "class": "InitialWorkDirRequirement",
 579                              "listing": [
 580                                  {
 581                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 582                                      "entryname": "example.conf"
 583                                  }
 584                              ]
 585                          }
 586                      ]})
 587         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 588                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 589         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 590                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 591                                                  metadata={"cwlVersion": "v1.0"})
 592         arvtool.formatgraph = None
 593
 594         job_order = {"pw": "blorp"}
 595         runner.secret_store.store(["pw"], job_order)
 596
 597         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets",
 598                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 599             j.run(enable_reuse=True, priority=500)
 600             runner.api.container_requests().create.assert_called_with(
 601                 body=JsonDiffMatcher({
 602                     'environment': {
 603                         'HOME': '/var/spool/cwl',
 604                         'TMPDIR': '/tmp'
 605                     },
 606                     'name': 'test_secrets',
 607                     'runtime_constraints': {
 608                         'vcpus': 1,
 609                         'ram': 1073741824
 610                     },
 611                     'use_existing': True,
 612                     'priority': 500,
 613                     'mounts': {
 614                         '/tmp': {'kind': 'tmp',
 615                                  "capacity": 1073741824
 616                              },
 617                         '/var/spool/cwl': {'kind': 'tmp',
 618                                            "capacity": 1073741824 }
 619                     },
 620                     'state': 'Committed',
 621                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 622                     'output_path': '/var/spool/cwl',
 623                     'output_ttl': 0,
 624                     'container_image': 'arvados/jobs',
 625                     'command': ['md5sum', 'example.conf'],
 626                     'cwd': '/var/spool/cwl',
 627                     'scheduling_parameters': {},
 628                     'properties': {},
 629                     "secret_mounts": {
 630                         "/var/spool/cwl/example.conf": {
 631                             "content": "username: user\npassword: blorp\n",
 632                             "kind": "text"
 633                         }
 634                     }
 635                 }))