sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def helper(self, runner, enable_reuse=True):
  60         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
  61
  62         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  63                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  64         loadingContext = arvados_cwl.context.ArvLoadingContext(
  65             {"avsc_names": avsc_names,
  66              "basedir": "",
  67              "make_fs_access": make_fs_access,
  68              "loader": Loader({}),
  69              "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  70         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  71             {"work_api": "containers",
  72              "basedir": "",
  73              "name": "test_run_"+str(enable_reuse),
  74              "make_fs_access": make_fs_access,
  75              "tmpdir": "/tmp",
  76              "enable_reuse": enable_reuse,
  77              "priority": 500,
  78              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  79             })
  80
  81         return loadingContext, runtimeContext
  82
  83     # Helper function to set up the ArvCwlExecutor to use the containers api
  84     # and test that the RuntimeStatusLoggingHandler is set up correctly
  85     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  86         api = mock.MagicMock()
  87         api._rootDesc = copy.deepcopy(get_rootDesc())
  88         del api._rootDesc.get('resources')['jobs']['methods']['create']
  89
  90         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  91         # adds the logging handler that will call runtime_status_update() mock
  92         self.assertFalse(gcc_mock.called)
  93         runner = arvados_cwl.ArvCwlExecutor(api)
  94         self.assertEqual(runner.work_api, 'containers')
  95         root_logger = logging.getLogger('')
  96         handlerClasses = [h.__class__ for h in root_logger.handlers]
  97         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
  98         return runner
  99
 100     # The test passes no builder.resources
 101     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 102     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 103     def test_run(self, keepdocker):
 104         for enable_reuse in (True, False):
 105             arv_docker_clear_cache()
 106
 107             runner = mock.MagicMock()
 108             runner.ignore_docker_for_reuse = False
 109             runner.intermediate_output_ttl = 0
 110             runner.secret_store = cwltool.secrets.SecretStore()
 111
 112             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 113             runner.api.collections().get().execute.return_value = {
 114                 "portable_data_hash": "99999999999999999999999999999993+99"}
 115
 116             tool = cmap({
 117                 "inputs": [],
 118                 "outputs": [],
 119                 "baseCommand": "ls",
 120                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 121                 "id": "#",
 122                 "class": "CommandLineTool"
 123             })
 124
 125             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 126
 127             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 128             arvtool.formatgraph = None
 129
 130             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 131                 j.run(runtimeContext)
 132                 runner.api.container_requests().create.assert_called_with(
 133                     body=JsonDiffMatcher({
 134                         'environment': {
 135                             'HOME': '/var/spool/cwl',
 136                             'TMPDIR': '/tmp'
 137                         },
 138                         'name': 'test_run_'+str(enable_reuse),
 139                         'runtime_constraints': {
 140                             'vcpus': 1,
 141                             'ram': 1073741824
 142                         },
 143                         'use_existing': enable_reuse,
 144                         'priority': 500,
 145                         'mounts': {
 146                             '/tmp': {'kind': 'tmp',
 147                                      "capacity": 1073741824
 148                                  },
 149                             '/var/spool/cwl': {'kind': 'tmp',
 150                                                "capacity": 1073741824 }
 151                         },
 152                         'state': 'Committed',
 153                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 154                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 155                         'output_path': '/var/spool/cwl',
 156                         'output_ttl': 0,
 157                         'container_image': '99999999999999999999999999999993+99',
 158                         'command': ['ls', '/var/spool/cwl'],
 159                         'cwd': '/var/spool/cwl',
 160                         'scheduling_parameters': {},
 161                         'properties': {},
 162                         'secret_mounts': {}
 163                     }))
 164
 165     # The test passes some fields in builder.resources
 166     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 167     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 168     def test_resource_requirements(self, keepdocker):
 169         arv_docker_clear_cache()
 170         runner = mock.MagicMock()
 171         runner.ignore_docker_for_reuse = False
 172         runner.intermediate_output_ttl = 3600
 173         runner.secret_store = cwltool.secrets.SecretStore()
 174
 175         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 176         runner.api.collections().get().execute.return_value = {
 177             "portable_data_hash": "99999999999999999999999999999993+99"}
 178
 179         tool = cmap({
 180             "inputs": [],
 181             "outputs": [],
 182             "hints": [{
 183                 "class": "ResourceRequirement",
 184                 "coresMin": 3,
 185                 "ramMin": 3000,
 186                 "tmpdirMin": 4000,
 187                 "outdirMin": 5000
 188             }, {
 189                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 190                 "keep_cache": 512
 191             }, {
 192                 "class": "http://arvados.org/cwl#APIRequirement",
 193             }, {
 194                 "class": "http://arvados.org/cwl#PartitionRequirement",
 195                 "partition": "blurb"
 196             }, {
 197                 "class": "http://arvados.org/cwl#IntermediateOutput",
 198                 "outputTTL": 7200
 199             }, {
 200                 "class": "http://arvados.org/cwl#ReuseRequirement",
 201                 "enableReuse": False
 202             }],
 203             "baseCommand": "ls",
 204             "id": "#",
 205             "class": "CommandLineTool"
 206         })
 207
 208         loadingContext, runtimeContext = self.helper(runner)
 209         runtimeContext.name = "test_resource_requirements"
 210
 211         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 212         arvtool.formatgraph = None
 213         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 214             j.run(runtimeContext)
 215
 216         call_args, call_kwargs = runner.api.container_requests().create.call_args
 217
 218         call_body_expected = {
 219             'environment': {
 220                 'HOME': '/var/spool/cwl',
 221                 'TMPDIR': '/tmp'
 222             },
 223             'name': 'test_resource_requirements',
 224             'runtime_constraints': {
 225                 'vcpus': 3,
 226                 'ram': 3145728000,
 227                 'keep_cache_ram': 536870912,
 228                 'API': True
 229             },
 230             'use_existing': False,
 231             'priority': 500,
 232             'mounts': {
 233                 '/tmp': {'kind': 'tmp',
 234                          "capacity": 4194304000 },
 235                 '/var/spool/cwl': {'kind': 'tmp',
 236                                    "capacity": 5242880000 }
 237             },
 238             'state': 'Committed',
 239             'output_name': 'Output for step test_resource_requirements',
 240             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 241             'output_path': '/var/spool/cwl',
 242             'output_ttl': 7200,
 243             'container_image': '99999999999999999999999999999993+99',
 244             'command': ['ls'],
 245             'cwd': '/var/spool/cwl',
 246             'scheduling_parameters': {
 247                 'partitions': ['blurb']
 248             },
 249             'properties': {},
 250             'secret_mounts': {}
 251         }
 252
 253         call_body = call_kwargs.get('body', None)
 254         self.assertNotEqual(None, call_body)
 255         for key in call_body:
 256             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 257
 258
 259     # The test passes some fields in builder.resources
 260     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 261     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 262     @mock.patch("arvados.collection.Collection")
 263     def test_initial_work_dir(self, collection_mock, keepdocker):
 264         arv_docker_clear_cache()
 265         runner = mock.MagicMock()
 266         runner.ignore_docker_for_reuse = False
 267         runner.intermediate_output_ttl = 0
 268         runner.secret_store = cwltool.secrets.SecretStore()
 269
 270         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 271         runner.api.collections().get().execute.return_value = {
 272             "portable_data_hash": "99999999999999999999999999999993+99"}
 273
 274         sourcemock = mock.MagicMock()
 275         def get_collection_mock(p):
 276             if "/" in p:
 277                 return (sourcemock, p.split("/", 1)[1])
 278             else:
 279                 return (sourcemock, "")
 280         runner.fs_access.get_collection.side_effect = get_collection_mock
 281
 282         vwdmock = mock.MagicMock()
 283         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 284
 285         tool = cmap({
 286             "inputs": [],
 287             "outputs": [],
 288             "hints": [{
 289                 "class": "InitialWorkDirRequirement",
 290                 "listing": [{
 291                     "class": "File",
 292                     "basename": "foo",
 293                     "location": "keep:99999999999999999999999999999995+99/bar"
 294                 },
 295                 {
 296                     "class": "Directory",
 297                     "basename": "foo2",
 298                     "location": "keep:99999999999999999999999999999995+99"
 299                 },
 300                 {
 301                     "class": "File",
 302                     "basename": "filename",
 303                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 304                 },
 305                 {
 306                     "class": "Directory",
 307                     "basename": "subdir",
 308                     "location": "keep:99999999999999999999999999999995+99/subdir"
 309                 }                        ]
 310             }],
 311             "baseCommand": "ls",
 312             "id": "#",
 313             "class": "CommandLineTool"
 314         })
 315
 316         loadingContext, runtimeContext = self.helper(runner)
 317         runtimeContext.name = "test_initial_work_dir"
 318
 319         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 320         arvtool.formatgraph = None
 321         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 322             j.run(runtimeContext)
 323
 324         call_args, call_kwargs = runner.api.container_requests().create.call_args
 325
 326         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 327         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 328         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 329         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 330
 331         call_body_expected = {
 332             'environment': {
 333                 'HOME': '/var/spool/cwl',
 334                 'TMPDIR': '/tmp'
 335             },
 336             'name': 'test_initial_work_dir',
 337             'runtime_constraints': {
 338                 'vcpus': 1,
 339                 'ram': 1073741824
 340             },
 341             'use_existing': True,
 342             'priority': 500,
 343             'mounts': {
 344                 '/tmp': {'kind': 'tmp',
 345                          "capacity": 1073741824 },
 346                 '/var/spool/cwl': {'kind': 'tmp',
 347                                    "capacity": 1073741824 },
 348                 '/var/spool/cwl/foo': {
 349                     'kind': 'collection',
 350                     'path': 'foo',
 351                     'portable_data_hash': '99999999999999999999999999999996+99'
 352                 },
 353                 '/var/spool/cwl/foo2': {
 354                     'kind': 'collection',
 355                     'path': 'foo2',
 356                     'portable_data_hash': '99999999999999999999999999999996+99'
 357                 },
 358                 '/var/spool/cwl/filename': {
 359                     'kind': 'collection',
 360                     'path': 'filename',
 361                     'portable_data_hash': '99999999999999999999999999999996+99'
 362                 },
 363                 '/var/spool/cwl/subdir': {
 364                     'kind': 'collection',
 365                     'path': 'subdir',
 366                     'portable_data_hash': '99999999999999999999999999999996+99'
 367                 }
 368             },
 369             'state': 'Committed',
 370             'output_name': 'Output for step test_initial_work_dir',
 371             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 372             'output_path': '/var/spool/cwl',
 373             'output_ttl': 0,
 374             'container_image': '99999999999999999999999999999993+99',
 375             'command': ['ls'],
 376             'cwd': '/var/spool/cwl',
 377             'scheduling_parameters': {
 378             },
 379             'properties': {},
 380             'secret_mounts': {}
 381         }
 382
 383         call_body = call_kwargs.get('body', None)
 384         self.assertNotEqual(None, call_body)
 385         for key in call_body:
 386             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 387
 388
 389     # Test redirecting stdin/stdout/stderr
 390     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 391     def test_redirects(self, keepdocker):
 392         arv_docker_clear_cache()
 393
 394         runner = mock.MagicMock()
 395         runner.ignore_docker_for_reuse = False
 396         runner.intermediate_output_ttl = 0
 397         runner.secret_store = cwltool.secrets.SecretStore()
 398
 399         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 400         runner.api.collections().get().execute.return_value = {
 401             "portable_data_hash": "99999999999999999999999999999993+99"}
 402
 403         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 404
 405         tool = cmap({
 406             "inputs": [],
 407             "outputs": [],
 408             "baseCommand": "ls",
 409             "stdout": "stdout.txt",
 410             "stderr": "stderr.txt",
 411             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 412             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 413             "id": "#",
 414             "class": "CommandLineTool"
 415         })
 416
 417         loadingContext, runtimeContext = self.helper(runner)
 418         runtimeContext.name = "test_run_redirect"
 419
 420         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 421         arvtool.formatgraph = None
 422         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 423             j.run(runtimeContext)
 424             runner.api.container_requests().create.assert_called_with(
 425                 body=JsonDiffMatcher({
 426                     'environment': {
 427                         'HOME': '/var/spool/cwl',
 428                         'TMPDIR': '/tmp'
 429                     },
 430                     'name': 'test_run_redirect',
 431                     'runtime_constraints': {
 432                         'vcpus': 1,
 433                         'ram': 1073741824
 434                     },
 435                     'use_existing': True,
 436                     'priority': 500,
 437                     'mounts': {
 438                         '/tmp': {'kind': 'tmp',
 439                                  "capacity": 1073741824 },
 440                         '/var/spool/cwl': {'kind': 'tmp',
 441                                            "capacity": 1073741824 },
 442                         "stderr": {
 443                             "kind": "file",
 444                             "path": "/var/spool/cwl/stderr.txt"
 445                         },
 446                         "stdin": {
 447                             "kind": "collection",
 448                             "path": "file.txt",
 449                             "portable_data_hash": "99999999999999999999999999999996+99"
 450                         },
 451                         "stdout": {
 452                             "kind": "file",
 453                             "path": "/var/spool/cwl/stdout.txt"
 454                         },
 455                     },
 456                     'state': 'Committed',
 457                     "output_name": "Output for step test_run_redirect",
 458                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 459                     'output_path': '/var/spool/cwl',
 460                     'output_ttl': 0,
 461                     'container_image': '99999999999999999999999999999993+99',
 462                     'command': ['ls', '/var/spool/cwl'],
 463                     'cwd': '/var/spool/cwl',
 464                     'scheduling_parameters': {},
 465                     'properties': {},
 466                     'secret_mounts': {}
 467                 }))
 468
 469     @mock.patch("arvados.collection.Collection")
 470     def test_done(self, col):
 471         api = mock.MagicMock()
 472
 473         runner = mock.MagicMock()
 474         runner.api = api
 475         runner.num_retries = 0
 476         runner.ignore_docker_for_reuse = False
 477         runner.intermediate_output_ttl = 0
 478         runner.secret_store = cwltool.secrets.SecretStore()
 479
 480         runner.api.containers().get().execute.return_value = {"state":"Complete",
 481                                                               "output": "abc+123",
 482                                                               "exit_code": 0}
 483
 484         col().open.return_value = []
 485
 486         loadingContext, runtimeContext = self.helper(runner)
 487
 488         arvjob = arvados_cwl.ArvadosContainer(runner,
 489                                               runtimeContext,
 490                                               mock.MagicMock(),
 491                                               {},
 492                                               None,
 493                                               [],
 494                                               [],
 495                                               "testjob")
 496         arvjob.output_callback = mock.MagicMock()
 497         arvjob.collect_outputs = mock.MagicMock()
 498         arvjob.successCodes = [0]
 499         arvjob.outdir = "/var/spool/cwl"
 500         arvjob.output_ttl = 3600
 501
 502         arvjob.collect_outputs.return_value = {"out": "stuff"}
 503
 504         arvjob.done({
 505             "state": "Final",
 506             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 507             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 508             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 509             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 510             "modified_at": "2017-05-26T12:01:22Z"
 511         })
 512
 513         self.assertFalse(api.collections().create.called)
 514         self.assertFalse(runner.runtime_status_error.called)
 515
 516         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 517         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 518         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 519
 520     # Test to make sure we dont call runtime_status_update if we already did
 521     # some where higher up in the call stack
 522     @mock.patch("arvados_cwl.util.get_current_container")
 523     def test_recursive_runtime_status_update(self, gcc_mock):
 524         self.setup_and_test_container_executor_and_logging(gcc_mock)
 525         root_logger = logging.getLogger('')
 526
 527         # get_current_container is invoked when we call runtime_status_update
 528         # so try and log again!
 529         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 530         try:
 531             root_logger.error("First Error")
 532         except RuntimeError:
 533             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 534
 535     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 536     @mock.patch("arvados_cwl.util.get_current_container")
 537     @mock.patch("arvados.collection.CollectionReader")
 538     @mock.patch("arvados.collection.Collection")
 539     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 540         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 541
 542         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 543         self.assertTrue(gcc_mock.called)
 544
 545         runner.num_retries = 0
 546         runner.ignore_docker_for_reuse = False
 547         runner.intermediate_output_ttl = 0
 548         runner.secret_store = cwltool.secrets.SecretStore()
 549         runner.label = mock.MagicMock()
 550         runner.label.return_value = '[container testjob]'
 551
 552         runner.api.containers().get().execute.return_value = {
 553             "state":"Complete",
 554             "output": "abc+123",
 555             "exit_code": 1,
 556             "log": "def+234"
 557         }
 558
 559         col().open.return_value = []
 560
 561         loadingContext, runtimeContext = self.helper(runner)
 562
 563         arvjob = arvados_cwl.ArvadosContainer(runner,
 564                                               runtimeContext,
 565                                               mock.MagicMock(),
 566                                               {},
 567                                               None,
 568                                               [],
 569                                               [],
 570                                               "testjob")
 571         arvjob.output_callback = mock.MagicMock()
 572         arvjob.collect_outputs = mock.MagicMock()
 573         arvjob.successCodes = [0]
 574         arvjob.outdir = "/var/spool/cwl"
 575         arvjob.output_ttl = 3600
 576         arvjob.collect_outputs.return_value = {"out": "stuff"}
 577
 578         arvjob.done({
 579             "state": "Final",
 580             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 581             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 582             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 583             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 584             "modified_at": "2017-05-26T12:01:22Z"
 585         })
 586
 587         rts_mock.assert_called_with(
 588             'error',
 589             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 590             '  ** log is empty **'
 591         )
 592         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 593
 594     # The test passes no builder.resources
 595     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 596     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 597     def test_mounts(self, keepdocker):
 598         arv_docker_clear_cache()
 599
 600         runner = mock.MagicMock()
 601         runner.ignore_docker_for_reuse = False
 602         runner.intermediate_output_ttl = 0
 603         runner.secret_store = cwltool.secrets.SecretStore()
 604
 605         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 606         runner.api.collections().get().execute.return_value = {
 607             "portable_data_hash": "99999999999999999999999999999994+99",
 608             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 609
 610         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 611
 612         tool = cmap({
 613             "inputs": [
 614                 {"id": "p1",
 615                  "type": "Directory"}
 616             ],
 617             "outputs": [],
 618             "baseCommand": "ls",
 619             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 620             "id": "#",
 621             "class": "CommandLineTool"
 622         })
 623
 624         loadingContext, runtimeContext = self.helper(runner)
 625         runtimeContext.name = "test_run_mounts"
 626
 627         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 628         arvtool.formatgraph = None
 629         job_order = {
 630             "p1": {
 631                 "class": "Directory",
 632                 "location": "keep:99999999999999999999999999999994+44",
 633                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 634                 "listing": [
 635                     {
 636                         "class": "File",
 637                         "location": "keep:99999999999999999999999999999994+44/file1",
 638                     },
 639                     {
 640                         "class": "File",
 641                         "location": "keep:99999999999999999999999999999994+44/file2",
 642                     }
 643                 ]
 644             }
 645         }
 646         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 647             j.run(runtimeContext)
 648             runner.api.container_requests().create.assert_called_with(
 649                 body=JsonDiffMatcher({
 650                     'environment': {
 651                         'HOME': '/var/spool/cwl',
 652                         'TMPDIR': '/tmp'
 653                     },
 654                     'name': 'test_run_mounts',
 655                     'runtime_constraints': {
 656                         'vcpus': 1,
 657                         'ram': 1073741824
 658                     },
 659                     'use_existing': True,
 660                     'priority': 500,
 661                     'mounts': {
 662                         "/keep/99999999999999999999999999999994+44": {
 663                             "kind": "collection",
 664                             "portable_data_hash": "99999999999999999999999999999994+44",
 665                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 666                         },
 667                         '/tmp': {'kind': 'tmp',
 668                                  "capacity": 1073741824 },
 669                         '/var/spool/cwl': {'kind': 'tmp',
 670                                            "capacity": 1073741824 }
 671                     },
 672                     'state': 'Committed',
 673                     'output_name': 'Output for step test_run_mounts',
 674                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 675                     'output_path': '/var/spool/cwl',
 676                     'output_ttl': 0,
 677                     'container_image': '99999999999999999999999999999994+99',
 678                     'command': ['ls', '/var/spool/cwl'],
 679                     'cwd': '/var/spool/cwl',
 680                     'scheduling_parameters': {},
 681                     'properties': {},
 682                     'secret_mounts': {}
 683                 }))
 684
 685     # The test passes no builder.resources
 686     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 687     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 688     def test_secrets(self, keepdocker):
 689         arv_docker_clear_cache()
 690
 691         runner = mock.MagicMock()
 692         runner.ignore_docker_for_reuse = False
 693         runner.intermediate_output_ttl = 0
 694         runner.secret_store = cwltool.secrets.SecretStore()
 695
 696         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 697         runner.api.collections().get().execute.return_value = {
 698             "portable_data_hash": "99999999999999999999999999999993+99"}
 699
 700         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
 701
 702         tool = cmap({"arguments": ["md5sum", "example.conf"],
 703                      "class": "CommandLineTool",
 704                      "hints": [
 705                          {
 706                              "class": "http://commonwl.org/cwltool#Secrets",
 707                              "secrets": [
 708                                  "#secret_job.cwl/pw"
 709                              ]
 710                          }
 711                      ],
 712                      "id": "#secret_job.cwl",
 713                      "inputs": [
 714                          {
 715                              "id": "#secret_job.cwl/pw",
 716                              "type": "string"
 717                          }
 718                      ],
 719                      "outputs": [
 720                      ],
 721                      "requirements": [
 722                          {
 723                              "class": "InitialWorkDirRequirement",
 724                              "listing": [
 725                                  {
 726                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 727                                      "entryname": "example.conf"
 728                                  }
 729                              ]
 730                          }
 731                      ]})
 732
 733         loadingContext, runtimeContext = self.helper(runner)
 734         runtimeContext.name = "test_secrets"
 735
 736         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 737         arvtool.formatgraph = None
 738
 739         job_order = {"pw": "blorp"}
 740         runner.secret_store.store(["pw"], job_order)
 741
 742         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 743             j.run(runtimeContext)
 744             runner.api.container_requests().create.assert_called_with(
 745                 body=JsonDiffMatcher({
 746                     'environment': {
 747                         'HOME': '/var/spool/cwl',
 748                         'TMPDIR': '/tmp'
 749                     },
 750                     'name': 'test_secrets',
 751                     'runtime_constraints': {
 752                         'vcpus': 1,
 753                         'ram': 1073741824
 754                     },
 755                     'use_existing': True,
 756                     'priority': 500,
 757                     'mounts': {
 758                         '/tmp': {'kind': 'tmp',
 759                                  "capacity": 1073741824
 760                              },
 761                         '/var/spool/cwl': {'kind': 'tmp',
 762                                            "capacity": 1073741824 }
 763                     },
 764                     'state': 'Committed',
 765                     'output_name': 'Output for step test_secrets',
 766                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 767                     'output_path': '/var/spool/cwl',
 768                     'output_ttl': 0,
 769                     'container_image': '99999999999999999999999999999993+99',
 770                     'command': ['md5sum', 'example.conf'],
 771                     'cwd': '/var/spool/cwl',
 772                     'scheduling_parameters': {},
 773                     'properties': {},
 774                     "secret_mounts": {
 775                         "/var/spool/cwl/example.conf": {
 776                             "content": "username: user\npassword: blorp\n",
 777                             "kind": "text"
 778                         }
 779                     }
 780                 }))
 781
 782     # The test passes no builder.resources
 783     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 784     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 785     def test_timelimit(self, keepdocker):
 786         arv_docker_clear_cache()
 787
 788         runner = mock.MagicMock()
 789         runner.ignore_docker_for_reuse = False
 790         runner.intermediate_output_ttl = 0
 791         runner.secret_store = cwltool.secrets.SecretStore()
 792
 793         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 794         runner.api.collections().get().execute.return_value = {
 795             "portable_data_hash": "99999999999999999999999999999993+99"}
 796
 797         tool = cmap({
 798             "inputs": [],
 799             "outputs": [],
 800             "baseCommand": "ls",
 801             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 802             "id": "#",
 803             "class": "CommandLineTool",
 804             "hints": [
 805                 {
 806                     "class": "ToolTimeLimit",
 807                     "timelimit": 42
 808                 }
 809             ]
 810         })
 811
 812         loadingContext, runtimeContext = self.helper(runner)
 813         runtimeContext.name = "test_timelimit"
 814
 815         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 816         arvtool.formatgraph = None
 817
 818         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 819             j.run(runtimeContext)
 820
 821         _, kwargs = runner.api.container_requests().create.call_args
 822         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))