sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import datetime
  12 import functools
  13 import cwltool.process
  14 import cwltool.secrets
  15 from schema_salad.ref_resolver import Loader
  16 from schema_salad.sourceline import cmap
  17
  18 from .matcher import JsonDiffMatcher
  19
  20 if not os.getenv('ARVADOS_DEBUG'):
  21     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  22     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  23
  24 class MockDateTime(datetime.datetime):
  25     @classmethod
  26     def now(cls):
  27         return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
  28
  29 datetime.datetime = MockDateTime
  30
  31 class TestContainer(unittest.TestCase):
  32
  33     # The test passes no builder.resources
  34     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  35     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  36     def test_run(self, keepdocker):
  37         for enable_reuse in (True, False):
  38             arv_docker_clear_cache()
  39
  40             runner = mock.MagicMock()
  41             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  42             runner.ignore_docker_for_reuse = False
  43             runner.intermediate_output_ttl = 0
  44             runner.secret_store = cwltool.secrets.SecretStore()
  45
  46             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  47             runner.api.collections().get().execute.return_value = {
  48                 "portable_data_hash": "99999999999999999999999999999993+99"}
  49
  50             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  51
  52             tool = cmap({
  53                 "inputs": [],
  54                 "outputs": [],
  55                 "baseCommand": "ls",
  56                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  57                 "id": "#",
  58                 "class": "CommandLineTool"
  59             })
  60             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  61                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  62             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  63                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}),
  64                                                      metadata={"cwlVersion": "v1.0"})
  65             arvtool.formatgraph = None
  66             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  67                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  68                 j.run(enable_reuse=enable_reuse, priority=500)
  69                 runner.api.container_requests().create.assert_called_with(
  70                     body=JsonDiffMatcher({
  71                         'environment': {
  72                             'HOME': '/var/spool/cwl',
  73                             'TMPDIR': '/tmp'
  74                         },
  75                         'name': 'test_run_'+str(enable_reuse),
  76                         'runtime_constraints': {
  77                             'vcpus': 1,
  78                             'ram': 1073741824
  79                         },
  80                         'use_existing': enable_reuse,
  81                         'priority': 500,
  82                         'mounts': {
  83                             '/tmp': {'kind': 'tmp',
  84                                      "capacity": 1073741824
  85                                  },
  86                             '/var/spool/cwl': {'kind': 'tmp',
  87                                                "capacity": 1073741824 }
  88                         },
  89                         'state': 'Committed',
  90                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  91                         'output_path': '/var/spool/cwl',
  92                         'output_ttl': 0,
  93                         'container_image': 'arvados/jobs',
  94                         'command': ['ls', '/var/spool/cwl'],
  95                         'cwd': '/var/spool/cwl',
  96                         'scheduling_parameters': {},
  97                         'properties': {},
  98                         'secret_mounts': {}
  99                     }))
 100
 101     # The test passes some fields in builder.resources
 102     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 103     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 104     def test_resource_requirements(self, keepdocker):
 105         arv_docker_clear_cache()
 106         runner = mock.MagicMock()
 107         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 108         runner.ignore_docker_for_reuse = False
 109         runner.intermediate_output_ttl = 3600
 110         runner.secret_store = cwltool.secrets.SecretStore()
 111
 112         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 113
 114         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 115         runner.api.collections().get().execute.return_value = {
 116             "portable_data_hash": "99999999999999999999999999999993+99"}
 117
 118         tool = cmap({
 119             "inputs": [],
 120             "outputs": [],
 121             "hints": [{
 122                 "class": "ResourceRequirement",
 123                 "coresMin": 3,
 124                 "ramMin": 3000,
 125                 "tmpdirMin": 4000,
 126                 "outdirMin": 5000
 127             }, {
 128                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 129                 "keep_cache": 512
 130             }, {
 131                 "class": "http://arvados.org/cwl#APIRequirement",
 132             }, {
 133                 "class": "http://arvados.org/cwl#PartitionRequirement",
 134                 "partition": "blurb"
 135             }, {
 136                 "class": "http://arvados.org/cwl#IntermediateOutput",
 137                 "outputTTL": 7200
 138             }, {
 139                 "class": "http://arvados.org/cwl#ReuseRequirement",
 140                 "enableReuse": False
 141             }],
 142             "baseCommand": "ls",
 143             "id": "#",
 144             "class": "CommandLineTool"
 145         })
 146         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 147                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 148         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 149                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 150                                                  loader=Loader({}), metadata={"cwlVersion": "v1.0"})
 151         arvtool.formatgraph = None
 152         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 153                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 154             j.run(enable_reuse=True, priority=500)
 155
 156         call_args, call_kwargs = runner.api.container_requests().create.call_args
 157
 158         call_body_expected = {
 159             'environment': {
 160                 'HOME': '/var/spool/cwl',
 161                 'TMPDIR': '/tmp'
 162             },
 163             'name': 'test_resource_requirements',
 164             'runtime_constraints': {
 165                 'vcpus': 3,
 166                 'ram': 3145728000,
 167                 'keep_cache_ram': 536870912,
 168                 'API': True
 169             },
 170             'use_existing': False,
 171             'priority': 500,
 172             'mounts': {
 173                 '/tmp': {'kind': 'tmp',
 174                          "capacity": 4194304000 },
 175                 '/var/spool/cwl': {'kind': 'tmp',
 176                                    "capacity": 5242880000 }
 177             },
 178             'state': 'Committed',
 179             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 180             'output_path': '/var/spool/cwl',
 181             'output_ttl': 7200,
 182             'container_image': 'arvados/jobs',
 183             'command': ['ls'],
 184             'cwd': '/var/spool/cwl',
 185             'scheduling_parameters': {
 186                 'partitions': ['blurb']
 187             },
 188             'properties': {},
 189             'secret_mounts': {}
 190         }
 191
 192         call_body = call_kwargs.get('body', None)
 193         self.assertNotEqual(None, call_body)
 194         for key in call_body:
 195             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 196
 197
 198     # The test passes some fields in builder.resources
 199     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 200     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 201     @mock.patch("arvados.collection.Collection")
 202     def test_initial_work_dir(self, collection_mock, keepdocker):
 203         arv_docker_clear_cache()
 204         runner = mock.MagicMock()
 205         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 206         runner.ignore_docker_for_reuse = False
 207         runner.intermediate_output_ttl = 0
 208         runner.secret_store = cwltool.secrets.SecretStore()
 209
 210         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 211
 212         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 213         runner.api.collections().get().execute.return_value = {
 214             "portable_data_hash": "99999999999999999999999999999993+99"}
 215
 216         sourcemock = mock.MagicMock()
 217         def get_collection_mock(p):
 218             if "/" in p:
 219                 return (sourcemock, p.split("/", 1)[1])
 220             else:
 221                 return (sourcemock, "")
 222         runner.fs_access.get_collection.side_effect = get_collection_mock
 223
 224         vwdmock = mock.MagicMock()
 225         collection_mock.return_value = vwdmock
 226         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 227
 228         tool = cmap({
 229             "inputs": [],
 230             "outputs": [],
 231             "hints": [{
 232                 "class": "InitialWorkDirRequirement",
 233                 "listing": [{
 234                     "class": "File",
 235                     "basename": "foo",
 236                     "location": "keep:99999999999999999999999999999995+99/bar"
 237                 },
 238                 {
 239                     "class": "Directory",
 240                     "basename": "foo2",
 241                     "location": "keep:99999999999999999999999999999995+99"
 242                 },
 243                 {
 244                     "class": "File",
 245                     "basename": "filename",
 246                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 247                 },
 248                 {
 249                     "class": "Directory",
 250                     "basename": "subdir",
 251                     "location": "keep:99999999999999999999999999999995+99/subdir"
 252                 }                        ]
 253             }],
 254             "baseCommand": "ls",
 255             "id": "#",
 256             "class": "CommandLineTool"
 257         })
 258         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 259                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 260         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 261                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 262                                                  loader=Loader({}), metadata={"cwlVersion": "v1.0"})
 263         arvtool.formatgraph = None
 264         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 265                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 266             j.run(priority=500)
 267
 268         call_args, call_kwargs = runner.api.container_requests().create.call_args
 269
 270         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 271         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 272         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 273         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 274
 275         call_body_expected = {
 276             'environment': {
 277                 'HOME': '/var/spool/cwl',
 278                 'TMPDIR': '/tmp'
 279             },
 280             'name': 'test_initial_work_dir',
 281             'runtime_constraints': {
 282                 'vcpus': 1,
 283                 'ram': 1073741824
 284             },
 285             'use_existing': True,
 286             'priority': 500,
 287             'mounts': {
 288                 '/tmp': {'kind': 'tmp',
 289                          "capacity": 1073741824 },
 290                 '/var/spool/cwl': {'kind': 'tmp',
 291                                    "capacity": 1073741824 },
 292                 '/var/spool/cwl/foo': {
 293                     'kind': 'collection',
 294                     'path': 'foo',
 295                     'portable_data_hash': '99999999999999999999999999999996+99'
 296                 },
 297                 '/var/spool/cwl/foo2': {
 298                     'kind': 'collection',
 299                     'path': 'foo2',
 300                     'portable_data_hash': '99999999999999999999999999999996+99'
 301                 },
 302                 '/var/spool/cwl/filename': {
 303                     'kind': 'collection',
 304                     'path': 'filename',
 305                     'portable_data_hash': '99999999999999999999999999999996+99'
 306                 },
 307                 '/var/spool/cwl/subdir': {
 308                     'kind': 'collection',
 309                     'path': 'subdir',
 310                     'portable_data_hash': '99999999999999999999999999999996+99'
 311                 }
 312             },
 313             'state': 'Committed',
 314             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 315             'output_path': '/var/spool/cwl',
 316             'output_ttl': 0,
 317             'container_image': 'arvados/jobs',
 318             'command': ['ls'],
 319             'cwd': '/var/spool/cwl',
 320             'scheduling_parameters': {
 321             },
 322             'properties': {},
 323             'secret_mounts': {}
 324         }
 325
 326         call_body = call_kwargs.get('body', None)
 327         self.assertNotEqual(None, call_body)
 328         for key in call_body:
 329             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 330
 331
 332     # Test redirecting stdin/stdout/stderr
 333     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 334     def test_redirects(self, keepdocker):
 335         arv_docker_clear_cache()
 336
 337         runner = mock.MagicMock()
 338         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 339         runner.ignore_docker_for_reuse = False
 340         runner.intermediate_output_ttl = 0
 341         runner.secret_store = cwltool.secrets.SecretStore()
 342
 343         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 344         runner.api.collections().get().execute.return_value = {
 345             "portable_data_hash": "99999999999999999999999999999993+99"}
 346
 347         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 348
 349         tool = cmap({
 350             "inputs": [],
 351             "outputs": [],
 352             "baseCommand": "ls",
 353             "stdout": "stdout.txt",
 354             "stderr": "stderr.txt",
 355             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 356             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 357             "id": "#",
 358             "class": "CommandLineTool"
 359         })
 360         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 361                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 362         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 363                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 364                                                  metadata={"cwlVersion": "v1.0"})
 365         arvtool.formatgraph = None
 366         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 367                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 368             j.run(priority=500)
 369             runner.api.container_requests().create.assert_called_with(
 370                 body=JsonDiffMatcher({
 371                     'environment': {
 372                         'HOME': '/var/spool/cwl',
 373                         'TMPDIR': '/tmp'
 374                     },
 375                     'name': 'test_run_redirect',
 376                     'runtime_constraints': {
 377                         'vcpus': 1,
 378                         'ram': 1073741824
 379                     },
 380                     'use_existing': True,
 381                     'priority': 500,
 382                     'mounts': {
 383                         '/tmp': {'kind': 'tmp',
 384                                  "capacity": 1073741824 },
 385                         '/var/spool/cwl': {'kind': 'tmp',
 386                                            "capacity": 1073741824 },
 387                         "stderr": {
 388                             "kind": "file",
 389                             "path": "/var/spool/cwl/stderr.txt"
 390                         },
 391                         "stdin": {
 392                             "kind": "collection",
 393                             "path": "file.txt",
 394                             "portable_data_hash": "99999999999999999999999999999996+99"
 395                         },
 396                         "stdout": {
 397                             "kind": "file",
 398                             "path": "/var/spool/cwl/stdout.txt"
 399                         },
 400                     },
 401                     'state': 'Committed',
 402                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 403                     'output_path': '/var/spool/cwl',
 404                     'output_ttl': 0,
 405                     'container_image': 'arvados/jobs',
 406                     'command': ['ls', '/var/spool/cwl'],
 407                     'cwd': '/var/spool/cwl',
 408                     'scheduling_parameters': {},
 409                     'properties': {},
 410                     'secret_mounts': {}
 411                 }))
 412
 413     @mock.patch("arvados.collection.Collection")
 414     def test_done(self, col):
 415         api = mock.MagicMock()
 416
 417         runner = mock.MagicMock()
 418         runner.api = api
 419         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 420         runner.num_retries = 0
 421         runner.ignore_docker_for_reuse = False
 422         runner.intermediate_output_ttl = 0
 423         runner.secret_store = cwltool.secrets.SecretStore()
 424
 425         runner.api.containers().get().execute.return_value = {"state":"Complete",
 426                                                               "output": "abc+123",
 427                                                               "exit_code": 0}
 428
 429         col().open.return_value = []
 430
 431         arvjob = arvados_cwl.ArvadosContainer(runner)
 432         arvjob.name = "testjob"
 433         arvjob.builder = mock.MagicMock()
 434         arvjob.output_callback = mock.MagicMock()
 435         arvjob.collect_outputs = mock.MagicMock()
 436         arvjob.successCodes = [0]
 437         arvjob.outdir = "/var/spool/cwl"
 438         arvjob.output_ttl = 3600
 439
 440         arvjob.collect_outputs.return_value = {"out": "stuff"}
 441
 442         arvjob.done({
 443             "state": "Final",
 444             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 445             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 446             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 447             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 448             "modified_at": "2017-05-26T12:01:22Z"
 449         })
 450
 451         self.assertFalse(api.collections().create.called)
 452
 453         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 454         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 455         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 456
 457     # The test passes no builder.resources
 458     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 459     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 460     def test_mounts(self, keepdocker):
 461         arv_docker_clear_cache()
 462
 463         runner = mock.MagicMock()
 464         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 465         runner.ignore_docker_for_reuse = False
 466         runner.intermediate_output_ttl = 0
 467         runner.secret_store = cwltool.secrets.SecretStore()
 468
 469         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 470         runner.api.collections().get().execute.return_value = {
 471             "portable_data_hash": "99999999999999999999999999999993+99"}
 472
 473         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 474
 475         tool = cmap({
 476             "inputs": [
 477                 {"id": "p1",
 478                  "type": "Directory"}
 479             ],
 480             "outputs": [],
 481             "baseCommand": "ls",
 482             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 483             "id": "#",
 484             "class": "CommandLineTool"
 485         })
 486         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 487                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 488         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 489                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 490                                                  metadata={"cwlVersion": "v1.0"})
 491         arvtool.formatgraph = None
 492         job_order = {
 493             "p1": {
 494                 "class": "Directory",
 495                 "location": "keep:99999999999999999999999999999994+44",
 496                 "listing": [
 497                     {
 498                         "class": "File",
 499                         "location": "keep:99999999999999999999999999999994+44/file1",
 500                     },
 501                     {
 502                         "class": "File",
 503                         "location": "keep:99999999999999999999999999999994+44/file2",
 504                     }
 505                 ]
 506             }
 507         }
 508         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 509                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 510             j.run(priority=500)
 511             runner.api.container_requests().create.assert_called_with(
 512                 body=JsonDiffMatcher({
 513                     'environment': {
 514                         'HOME': '/var/spool/cwl',
 515                         'TMPDIR': '/tmp'
 516                     },
 517                     'name': 'test_run_mounts',
 518                     'runtime_constraints': {
 519                         'vcpus': 1,
 520                         'ram': 1073741824
 521                     },
 522                     'use_existing': True,
 523                     'priority': 500,
 524                     'mounts': {
 525                         "/keep/99999999999999999999999999999994+44": {
 526                             "kind": "collection",
 527                             "portable_data_hash": "99999999999999999999999999999994+44"
 528                         },
 529                         '/tmp': {'kind': 'tmp',
 530                                  "capacity": 1073741824 },
 531                         '/var/spool/cwl': {'kind': 'tmp',
 532                                            "capacity": 1073741824 }
 533                     },
 534                     'state': 'Committed',
 535                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 536                     'output_path': '/var/spool/cwl',
 537                     'output_ttl': 0,
 538                     'container_image': 'arvados/jobs',
 539                     'command': ['ls', '/var/spool/cwl'],
 540                     'cwd': '/var/spool/cwl',
 541                     'scheduling_parameters': {},
 542                     'properties': {},
 543                     'secret_mounts': {}
 544                 }))
 545
 546     # The test passes no builder.resources
 547     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 548     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 549     def test_secrets(self, keepdocker):
 550         arv_docker_clear_cache()
 551
 552         runner = mock.MagicMock()
 553         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 554         runner.ignore_docker_for_reuse = False
 555         runner.intermediate_output_ttl = 0
 556         runner.secret_store = cwltool.secrets.SecretStore()
 557
 558         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 559         runner.api.collections().get().execute.return_value = {
 560             "portable_data_hash": "99999999999999999999999999999993+99"}
 561
 562         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 563
 564         tool = cmap({"arguments": ["md5sum", "example.conf"],
 565                      "class": "CommandLineTool",
 566                      "hints": [
 567                          {
 568                              "class": "http://commonwl.org/cwltool#Secrets",
 569                              "secrets": [
 570                                  "#secret_job.cwl/pw"
 571                              ]
 572                          }
 573                      ],
 574                      "id": "#secret_job.cwl",
 575                      "inputs": [
 576                          {
 577                              "id": "#secret_job.cwl/pw",
 578                              "type": "string"
 579                          }
 580                      ],
 581                      "outputs": [
 582                      ],
 583                      "requirements": [
 584                          {
 585                              "class": "InitialWorkDirRequirement",
 586                              "listing": [
 587                                  {
 588                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 589                                      "entryname": "example.conf"
 590                                  }
 591                              ]
 592                          }
 593                      ]})
 594         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 595                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 596         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 597                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}),
 598                                                  metadata={"cwlVersion": "v1.0"})
 599         arvtool.formatgraph = None
 600
 601         job_order = {"pw": "blorp"}
 602         runner.secret_store.store(["pw"], job_order)
 603
 604         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets",
 605                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 606             j.run(enable_reuse=True, priority=500)
 607             runner.api.container_requests().create.assert_called_with(
 608                 body=JsonDiffMatcher({
 609                     'environment': {
 610                         'HOME': '/var/spool/cwl',
 611                         'TMPDIR': '/tmp'
 612                     },
 613                     'name': 'test_secrets',
 614                     'runtime_constraints': {
 615                         'vcpus': 1,
 616                         'ram': 1073741824
 617                     },
 618                     'use_existing': True,
 619                     'priority': 500,
 620                     'mounts': {
 621                         '/tmp': {'kind': 'tmp',
 622                                  "capacity": 1073741824
 623                              },
 624                         '/var/spool/cwl': {'kind': 'tmp',
 625                                            "capacity": 1073741824 }
 626                     },
 627                     'state': 'Committed',
 628                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 629                     'output_path': '/var/spool/cwl',
 630                     'output_ttl': 0,
 631                     'container_image': 'arvados/jobs',
 632                     'command': ['md5sum', 'example.conf'],
 633                     'cwd': '/var/spool/cwl',
 634                     'scheduling_parameters': {},
 635                     'properties': {},
 636                     "secret_mounts": {
 637                         "/var/spool/cwl/example.conf": {
 638                             "content": "username: user\npassword: blorp\n",
 639                             "kind": "text"
 640                         }
 641                     }
 642                 }))
 643
 644     def test_get_intermediate_collection_info(self):
 645         arvrunner = mock.MagicMock()
 646         arvrunner.intermediate_output_ttl = 60
 647         arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
 648
 649         container = arvados_cwl.ArvadosContainer(arvrunner)
 650
 651         info = container._get_intermediate_collection_info()
 652
 653         self.assertEqual(info["name"], "Intermediate collection")
 654         self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
 655         self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})