sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def helper(self, runner, enable_reuse=True):
  60         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  61
  62         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  63                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  64         loadingContext = arvados_cwl.context.ArvLoadingContext(
  65             {"avsc_names": avsc_names,
  66              "basedir": "",
  67              "make_fs_access": make_fs_access,
  68              "loader": Loader({}),
  69              "metadata": {"cwlVersion": "v1.0"}})
  70         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  71             {"work_api": "containers",
  72              "basedir": "",
  73              "name": "test_run_"+str(enable_reuse),
  74              "make_fs_access": make_fs_access,
  75              "tmpdir": "/tmp",
  76              "enable_reuse": enable_reuse,
  77              "priority": 500,
  78              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  79             })
  80
  81         return loadingContext, runtimeContext
  82
  83     # The test passes no builder.resources
  84     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  85     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  86     def test_run(self, keepdocker):
  87         for enable_reuse in (True, False):
  88             arv_docker_clear_cache()
  89
  90             runner = mock.MagicMock()
  91             runner.ignore_docker_for_reuse = False
  92             runner.intermediate_output_ttl = 0
  93             runner.secret_store = cwltool.secrets.SecretStore()
  94
  95             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  96             runner.api.collections().get().execute.return_value = {
  97                 "portable_data_hash": "99999999999999999999999999999993+99"}
  98
  99             tool = cmap({
 100                 "inputs": [],
 101                 "outputs": [],
 102                 "baseCommand": "ls",
 103                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 104                 "id": "#",
 105                 "class": "CommandLineTool"
 106             })
 107
 108             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 109
 110             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 111             arvtool.formatgraph = None
 112
 113             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 114                 j.run(runtimeContext)
 115                 runner.api.container_requests().create.assert_called_with(
 116                     body=JsonDiffMatcher({
 117                         'environment': {
 118                             'HOME': '/var/spool/cwl',
 119                             'TMPDIR': '/tmp'
 120                         },
 121                         'name': 'test_run_'+str(enable_reuse),
 122                         'runtime_constraints': {
 123                             'vcpus': 1,
 124                             'ram': 1073741824
 125                         },
 126                         'use_existing': enable_reuse,
 127                         'priority': 500,
 128                         'mounts': {
 129                             '/tmp': {'kind': 'tmp',
 130                                      "capacity": 1073741824
 131                                  },
 132                             '/var/spool/cwl': {'kind': 'tmp',
 133                                                "capacity": 1073741824 }
 134                         },
 135                         'state': 'Committed',
 136                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 137                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 138                         'output_path': '/var/spool/cwl',
 139                         'output_ttl': 0,
 140                         'container_image': '99999999999999999999999999999993+99',
 141                         'command': ['ls', '/var/spool/cwl'],
 142                         'cwd': '/var/spool/cwl',
 143                         'scheduling_parameters': {},
 144                         'properties': {},
 145                         'secret_mounts': {}
 146                     }))
 147
 148     # The test passes some fields in builder.resources
 149     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 150     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 151     def test_resource_requirements(self, keepdocker):
 152         arv_docker_clear_cache()
 153         runner = mock.MagicMock()
 154         runner.ignore_docker_for_reuse = False
 155         runner.intermediate_output_ttl = 3600
 156         runner.secret_store = cwltool.secrets.SecretStore()
 157
 158         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 159         runner.api.collections().get().execute.return_value = {
 160             "portable_data_hash": "99999999999999999999999999999993+99"}
 161
 162         tool = cmap({
 163             "inputs": [],
 164             "outputs": [],
 165             "hints": [{
 166                 "class": "ResourceRequirement",
 167                 "coresMin": 3,
 168                 "ramMin": 3000,
 169                 "tmpdirMin": 4000,
 170                 "outdirMin": 5000
 171             }, {
 172                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 173                 "keep_cache": 512
 174             }, {
 175                 "class": "http://arvados.org/cwl#APIRequirement",
 176             }, {
 177                 "class": "http://arvados.org/cwl#PartitionRequirement",
 178                 "partition": "blurb"
 179             }, {
 180                 "class": "http://arvados.org/cwl#IntermediateOutput",
 181                 "outputTTL": 7200
 182             }, {
 183                 "class": "http://arvados.org/cwl#ReuseRequirement",
 184                 "enableReuse": False
 185             }],
 186             "baseCommand": "ls",
 187             "id": "#",
 188             "class": "CommandLineTool"
 189         })
 190
 191         loadingContext, runtimeContext = self.helper(runner)
 192         runtimeContext.name = "test_resource_requirements"
 193
 194         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 195         arvtool.formatgraph = None
 196         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 197             j.run(runtimeContext)
 198
 199         call_args, call_kwargs = runner.api.container_requests().create.call_args
 200
 201         call_body_expected = {
 202             'environment': {
 203                 'HOME': '/var/spool/cwl',
 204                 'TMPDIR': '/tmp'
 205             },
 206             'name': 'test_resource_requirements',
 207             'runtime_constraints': {
 208                 'vcpus': 3,
 209                 'ram': 3145728000,
 210                 'keep_cache_ram': 536870912,
 211                 'API': True
 212             },
 213             'use_existing': False,
 214             'priority': 500,
 215             'mounts': {
 216                 '/tmp': {'kind': 'tmp',
 217                          "capacity": 4194304000 },
 218                 '/var/spool/cwl': {'kind': 'tmp',
 219                                    "capacity": 5242880000 }
 220             },
 221             'state': 'Committed',
 222             'output_name': 'Output for step test_resource_requirements',
 223             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 224             'output_path': '/var/spool/cwl',
 225             'output_ttl': 7200,
 226             'container_image': '99999999999999999999999999999993+99',
 227             'command': ['ls'],
 228             'cwd': '/var/spool/cwl',
 229             'scheduling_parameters': {
 230                 'partitions': ['blurb']
 231             },
 232             'properties': {},
 233             'secret_mounts': {}
 234         }
 235
 236         call_body = call_kwargs.get('body', None)
 237         self.assertNotEqual(None, call_body)
 238         for key in call_body:
 239             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 240
 241
 242     # The test passes some fields in builder.resources
 243     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 244     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 245     @mock.patch("arvados.collection.Collection")
 246     def test_initial_work_dir(self, collection_mock, keepdocker):
 247         arv_docker_clear_cache()
 248         runner = mock.MagicMock()
 249         runner.ignore_docker_for_reuse = False
 250         runner.intermediate_output_ttl = 0
 251         runner.secret_store = cwltool.secrets.SecretStore()
 252
 253         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 254         runner.api.collections().get().execute.return_value = {
 255             "portable_data_hash": "99999999999999999999999999999993+99"}
 256
 257         sourcemock = mock.MagicMock()
 258         def get_collection_mock(p):
 259             if "/" in p:
 260                 return (sourcemock, p.split("/", 1)[1])
 261             else:
 262                 return (sourcemock, "")
 263         runner.fs_access.get_collection.side_effect = get_collection_mock
 264
 265         vwdmock = mock.MagicMock()
 266         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 267
 268         tool = cmap({
 269             "inputs": [],
 270             "outputs": [],
 271             "hints": [{
 272                 "class": "InitialWorkDirRequirement",
 273                 "listing": [{
 274                     "class": "File",
 275                     "basename": "foo",
 276                     "location": "keep:99999999999999999999999999999995+99/bar"
 277                 },
 278                 {
 279                     "class": "Directory",
 280                     "basename": "foo2",
 281                     "location": "keep:99999999999999999999999999999995+99"
 282                 },
 283                 {
 284                     "class": "File",
 285                     "basename": "filename",
 286                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 287                 },
 288                 {
 289                     "class": "Directory",
 290                     "basename": "subdir",
 291                     "location": "keep:99999999999999999999999999999995+99/subdir"
 292                 }                        ]
 293             }],
 294             "baseCommand": "ls",
 295             "id": "#",
 296             "class": "CommandLineTool"
 297         })
 298
 299         loadingContext, runtimeContext = self.helper(runner)
 300         runtimeContext.name = "test_initial_work_dir"
 301
 302         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 303         arvtool.formatgraph = None
 304         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 305             j.run(runtimeContext)
 306
 307         call_args, call_kwargs = runner.api.container_requests().create.call_args
 308
 309         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 310         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 311         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 312         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 313
 314         call_body_expected = {
 315             'environment': {
 316                 'HOME': '/var/spool/cwl',
 317                 'TMPDIR': '/tmp'
 318             },
 319             'name': 'test_initial_work_dir',
 320             'runtime_constraints': {
 321                 'vcpus': 1,
 322                 'ram': 1073741824
 323             },
 324             'use_existing': True,
 325             'priority': 500,
 326             'mounts': {
 327                 '/tmp': {'kind': 'tmp',
 328                          "capacity": 1073741824 },
 329                 '/var/spool/cwl': {'kind': 'tmp',
 330                                    "capacity": 1073741824 },
 331                 '/var/spool/cwl/foo': {
 332                     'kind': 'collection',
 333                     'path': 'foo',
 334                     'portable_data_hash': '99999999999999999999999999999996+99'
 335                 },
 336                 '/var/spool/cwl/foo2': {
 337                     'kind': 'collection',
 338                     'path': 'foo2',
 339                     'portable_data_hash': '99999999999999999999999999999996+99'
 340                 },
 341                 '/var/spool/cwl/filename': {
 342                     'kind': 'collection',
 343                     'path': 'filename',
 344                     'portable_data_hash': '99999999999999999999999999999996+99'
 345                 },
 346                 '/var/spool/cwl/subdir': {
 347                     'kind': 'collection',
 348                     'path': 'subdir',
 349                     'portable_data_hash': '99999999999999999999999999999996+99'
 350                 }
 351             },
 352             'state': 'Committed',
 353             'output_name': 'Output for step test_initial_work_dir',
 354             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 355             'output_path': '/var/spool/cwl',
 356             'output_ttl': 0,
 357             'container_image': '99999999999999999999999999999993+99',
 358             'command': ['ls'],
 359             'cwd': '/var/spool/cwl',
 360             'scheduling_parameters': {
 361             },
 362             'properties': {},
 363             'secret_mounts': {}
 364         }
 365
 366         call_body = call_kwargs.get('body', None)
 367         self.assertNotEqual(None, call_body)
 368         for key in call_body:
 369             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 370
 371
 372     # Test redirecting stdin/stdout/stderr
 373     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 374     def test_redirects(self, keepdocker):
 375         arv_docker_clear_cache()
 376
 377         runner = mock.MagicMock()
 378         runner.ignore_docker_for_reuse = False
 379         runner.intermediate_output_ttl = 0
 380         runner.secret_store = cwltool.secrets.SecretStore()
 381
 382         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 383         runner.api.collections().get().execute.return_value = {
 384             "portable_data_hash": "99999999999999999999999999999993+99"}
 385
 386         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 387
 388         tool = cmap({
 389             "inputs": [],
 390             "outputs": [],
 391             "baseCommand": "ls",
 392             "stdout": "stdout.txt",
 393             "stderr": "stderr.txt",
 394             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 395             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 396             "id": "#",
 397             "class": "CommandLineTool"
 398         })
 399
 400         loadingContext, runtimeContext = self.helper(runner)
 401         runtimeContext.name = "test_run_redirect"
 402
 403         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 404         arvtool.formatgraph = None
 405         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 406             j.run(runtimeContext)
 407             runner.api.container_requests().create.assert_called_with(
 408                 body=JsonDiffMatcher({
 409                     'environment': {
 410                         'HOME': '/var/spool/cwl',
 411                         'TMPDIR': '/tmp'
 412                     },
 413                     'name': 'test_run_redirect',
 414                     'runtime_constraints': {
 415                         'vcpus': 1,
 416                         'ram': 1073741824
 417                     },
 418                     'use_existing': True,
 419                     'priority': 500,
 420                     'mounts': {
 421                         '/tmp': {'kind': 'tmp',
 422                                  "capacity": 1073741824 },
 423                         '/var/spool/cwl': {'kind': 'tmp',
 424                                            "capacity": 1073741824 },
 425                         "stderr": {
 426                             "kind": "file",
 427                             "path": "/var/spool/cwl/stderr.txt"
 428                         },
 429                         "stdin": {
 430                             "kind": "collection",
 431                             "path": "file.txt",
 432                             "portable_data_hash": "99999999999999999999999999999996+99"
 433                         },
 434                         "stdout": {
 435                             "kind": "file",
 436                             "path": "/var/spool/cwl/stdout.txt"
 437                         },
 438                     },
 439                     'state': 'Committed',
 440                     "output_name": "Output for step test_run_redirect",
 441                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 442                     'output_path': '/var/spool/cwl',
 443                     'output_ttl': 0,
 444                     'container_image': '99999999999999999999999999999993+99',
 445                     'command': ['ls', '/var/spool/cwl'],
 446                     'cwd': '/var/spool/cwl',
 447                     'scheduling_parameters': {},
 448                     'properties': {},
 449                     'secret_mounts': {}
 450                 }))
 451
 452     @mock.patch("arvados.collection.Collection")
 453     def test_done(self, col):
 454         api = mock.MagicMock()
 455
 456         runner = mock.MagicMock()
 457         runner.api = api
 458         runner.num_retries = 0
 459         runner.ignore_docker_for_reuse = False
 460         runner.intermediate_output_ttl = 0
 461         runner.secret_store = cwltool.secrets.SecretStore()
 462
 463         runner.api.containers().get().execute.return_value = {"state":"Complete",
 464                                                               "output": "abc+123",
 465                                                               "exit_code": 0}
 466
 467         col().open.return_value = []
 468
 469         loadingContext, runtimeContext = self.helper(runner)
 470
 471         arvjob = arvados_cwl.ArvadosContainer(runner,
 472                                               runtimeContext,
 473                                               mock.MagicMock(),
 474                                               {},
 475                                               None,
 476                                               [],
 477                                               [],
 478                                               "testjob")
 479         arvjob.output_callback = mock.MagicMock()
 480         arvjob.collect_outputs = mock.MagicMock()
 481         arvjob.successCodes = [0]
 482         arvjob.outdir = "/var/spool/cwl"
 483         arvjob.output_ttl = 3600
 484
 485         arvjob.collect_outputs.return_value = {"out": "stuff"}
 486
 487         arvjob.done({
 488             "state": "Final",
 489             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 490             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 491             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 492             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 493             "modified_at": "2017-05-26T12:01:22Z"
 494         })
 495
 496         self.assertFalse(api.collections().create.called)
 497         self.assertFalse(runner.runtime_status_error.called)
 498
 499         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 500         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 501         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 502
 503     # Test for issue 14886
 504     # Test to make sure we dont call runtime_status_update if we already did
 505     # some where higher up in the call stack
 506     def test_runtime_status_update_api_failure(self):
 507         api = mock.MagicMock()
 508         api._rootDesc = copy.deepcopy(get_rootDesc())
 509         del api._rootDesc.get('resources')['jobs']['methods']['create']
 510
 511         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 512         # adds the logging handler that will call runtime_status_update()
 513         api.containers().current().execute.return_value = mock.MagicMock()
 514         runner = arvados_cwl.ArvCwlExecutor(api)
 515         self.assertEqual(runner.work_api, 'containers')
 516         root_logger = logging.getLogger('')
 517         handlerClasses = [h.__class__ for h in root_logger.handlers]
 518         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 519
 520         # api.containers() is invoked when we call root_logger.error, so try and log again!
 521         api.containers().current.side_effect = lambda : root_logger.error("Second Error")
 522         root_logger.error("First Error")
 523
 524     @mock.patch("arvados_cwl.util.get_current_container")
 525     @mock.patch("arvados.collection.CollectionReader")
 526     @mock.patch("arvados.collection.Collection")
 527     def test_child_failure(self, col, reader, gcc_mock):
 528         api = mock.MagicMock()
 529         api._rootDesc = copy.deepcopy(get_rootDesc())
 530         del api._rootDesc.get('resources')['jobs']['methods']['create']
 531
 532         # Set up runner with mocked runtime_status_update()
 533         self.assertFalse(gcc_mock.called)
 534         runtime_status_update = mock.MagicMock()
 535         arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update
 536         runner = arvados_cwl.ArvCwlExecutor(api)
 537         self.assertEqual(runner.work_api, 'containers')
 538
 539         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 540         # adds the logging handler that will call runtime_status_update() mock
 541         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 542         self.assertTrue(gcc_mock.called)
 543         root_logger = logging.getLogger('')
 544         handlerClasses = [h.__class__ for h in root_logger.handlers]
 545         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 546
 547         runner.num_retries = 0
 548         runner.ignore_docker_for_reuse = False
 549         runner.intermediate_output_ttl = 0
 550         runner.secret_store = cwltool.secrets.SecretStore()
 551         runner.label = mock.MagicMock()
 552         runner.label.return_value = '[container testjob]'
 553
 554         runner.api.containers().get().execute.return_value = {
 555             "state":"Complete",
 556             "output": "abc+123",
 557             "exit_code": 1,
 558             "log": "def+234"
 559         }
 560
 561         col().open.return_value = []
 562
 563         loadingContext, runtimeContext = self.helper(runner)
 564
 565         arvjob = arvados_cwl.ArvadosContainer(runner,
 566                                               runtimeContext,
 567                                               mock.MagicMock(),
 568                                               {},
 569                                               None,
 570                                               [],
 571                                               [],
 572                                               "testjob")
 573         arvjob.output_callback = mock.MagicMock()
 574         arvjob.collect_outputs = mock.MagicMock()
 575         arvjob.successCodes = [0]
 576         arvjob.outdir = "/var/spool/cwl"
 577         arvjob.output_ttl = 3600
 578         arvjob.collect_outputs.return_value = {"out": "stuff"}
 579
 580         arvjob.done({
 581             "state": "Final",
 582             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 583             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 584             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 585             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 586             "modified_at": "2017-05-26T12:01:22Z"
 587         })
 588
 589         runtime_status_update.assert_called_with(
 590             'error',
 591             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 592             '  ** log is empty **'
 593         )
 594         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 595
 596     # The test passes no builder.resources
 597     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 598     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 599     def test_mounts(self, keepdocker):
 600         arv_docker_clear_cache()
 601
 602         runner = mock.MagicMock()
 603         runner.ignore_docker_for_reuse = False
 604         runner.intermediate_output_ttl = 0
 605         runner.secret_store = cwltool.secrets.SecretStore()
 606
 607         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 608         runner.api.collections().get().execute.return_value = {
 609             "portable_data_hash": "99999999999999999999999999999994+99",
 610             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 611
 612         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 613
 614         tool = cmap({
 615             "inputs": [
 616                 {"id": "p1",
 617                  "type": "Directory"}
 618             ],
 619             "outputs": [],
 620             "baseCommand": "ls",
 621             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 622             "id": "#",
 623             "class": "CommandLineTool"
 624         })
 625
 626         loadingContext, runtimeContext = self.helper(runner)
 627         runtimeContext.name = "test_run_mounts"
 628
 629         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 630         arvtool.formatgraph = None
 631         job_order = {
 632             "p1": {
 633                 "class": "Directory",
 634                 "location": "keep:99999999999999999999999999999994+44",
 635                 "listing": [
 636                     {
 637                         "class": "File",
 638                         "location": "keep:99999999999999999999999999999994+44/file1",
 639                     },
 640                     {
 641                         "class": "File",
 642                         "location": "keep:99999999999999999999999999999994+44/file2",
 643                     }
 644                 ]
 645             }
 646         }
 647         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 648             j.run(runtimeContext)
 649             runner.api.container_requests().create.assert_called_with(
 650                 body=JsonDiffMatcher({
 651                     'environment': {
 652                         'HOME': '/var/spool/cwl',
 653                         'TMPDIR': '/tmp'
 654                     },
 655                     'name': 'test_run_mounts',
 656                     'runtime_constraints': {
 657                         'vcpus': 1,
 658                         'ram': 1073741824
 659                     },
 660                     'use_existing': True,
 661                     'priority': 500,
 662                     'mounts': {
 663                         "/keep/99999999999999999999999999999994+44": {
 664                             "kind": "collection",
 665                             "portable_data_hash": "99999999999999999999999999999994+44"
 666                         },
 667                         '/tmp': {'kind': 'tmp',
 668                                  "capacity": 1073741824 },
 669                         '/var/spool/cwl': {'kind': 'tmp',
 670                                            "capacity": 1073741824 }
 671                     },
 672                     'state': 'Committed',
 673                     'output_name': 'Output for step test_run_mounts',
 674                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 675                     'output_path': '/var/spool/cwl',
 676                     'output_ttl': 0,
 677                     'container_image': '99999999999999999999999999999994+99',
 678                     'command': ['ls', '/var/spool/cwl'],
 679                     'cwd': '/var/spool/cwl',
 680                     'scheduling_parameters': {},
 681                     'properties': {},
 682                     'secret_mounts': {}
 683                 }))
 684
 685     # The test passes no builder.resources
 686     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 687     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 688     def test_secrets(self, keepdocker):
 689         arv_docker_clear_cache()
 690
 691         runner = mock.MagicMock()
 692         runner.ignore_docker_for_reuse = False
 693         runner.intermediate_output_ttl = 0
 694         runner.secret_store = cwltool.secrets.SecretStore()
 695
 696         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 697         runner.api.collections().get().execute.return_value = {
 698             "portable_data_hash": "99999999999999999999999999999993+99"}
 699
 700         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 701
 702         tool = cmap({"arguments": ["md5sum", "example.conf"],
 703                      "class": "CommandLineTool",
 704                      "hints": [
 705                          {
 706                              "class": "http://commonwl.org/cwltool#Secrets",
 707                              "secrets": [
 708                                  "#secret_job.cwl/pw"
 709                              ]
 710                          }
 711                      ],
 712                      "id": "#secret_job.cwl",
 713                      "inputs": [
 714                          {
 715                              "id": "#secret_job.cwl/pw",
 716                              "type": "string"
 717                          }
 718                      ],
 719                      "outputs": [
 720                      ],
 721                      "requirements": [
 722                          {
 723                              "class": "InitialWorkDirRequirement",
 724                              "listing": [
 725                                  {
 726                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 727                                      "entryname": "example.conf"
 728                                  }
 729                              ]
 730                          }
 731                      ]})
 732
 733         loadingContext, runtimeContext = self.helper(runner)
 734         runtimeContext.name = "test_secrets"
 735
 736         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 737         arvtool.formatgraph = None
 738
 739         job_order = {"pw": "blorp"}
 740         runner.secret_store.store(["pw"], job_order)
 741
 742         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 743             j.run(runtimeContext)
 744             runner.api.container_requests().create.assert_called_with(
 745                 body=JsonDiffMatcher({
 746                     'environment': {
 747                         'HOME': '/var/spool/cwl',
 748                         'TMPDIR': '/tmp'
 749                     },
 750                     'name': 'test_secrets',
 751                     'runtime_constraints': {
 752                         'vcpus': 1,
 753                         'ram': 1073741824
 754                     },
 755                     'use_existing': True,
 756                     'priority': 500,
 757                     'mounts': {
 758                         '/tmp': {'kind': 'tmp',
 759                                  "capacity": 1073741824
 760                              },
 761                         '/var/spool/cwl': {'kind': 'tmp',
 762                                            "capacity": 1073741824 }
 763                     },
 764                     'state': 'Committed',
 765                     'output_name': 'Output for step test_secrets',
 766                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 767                     'output_path': '/var/spool/cwl',
 768                     'output_ttl': 0,
 769                     'container_image': '99999999999999999999999999999993+99',
 770                     'command': ['md5sum', 'example.conf'],
 771                     'cwd': '/var/spool/cwl',
 772                     'scheduling_parameters': {},
 773                     'properties': {},
 774                     "secret_mounts": {
 775                         "/var/spool/cwl/example.conf": {
 776                             "content": "username: user\npassword: blorp\n",
 777                             "kind": "text"
 778                         }
 779                     }
 780                 }))
 781
 782     # The test passes no builder.resources
 783     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 784     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 785     def test_timelimit(self, keepdocker):
 786         arv_docker_clear_cache()
 787
 788         runner = mock.MagicMock()
 789         runner.ignore_docker_for_reuse = False
 790         runner.intermediate_output_ttl = 0
 791         runner.secret_store = cwltool.secrets.SecretStore()
 792
 793         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 794         runner.api.collections().get().execute.return_value = {
 795             "portable_data_hash": "99999999999999999999999999999993+99"}
 796
 797         tool = cmap({
 798             "inputs": [],
 799             "outputs": [],
 800             "baseCommand": "ls",
 801             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 802             "id": "#",
 803             "class": "CommandLineTool",
 804             "hints": [
 805                 {
 806                     "class": "http://commonwl.org/cwltool#TimeLimit",
 807                     "timelimit": 42
 808                 }
 809             ]
 810         })
 811
 812         loadingContext, runtimeContext = self.helper(runner)
 813         runtimeContext.name = "test_timelimit"
 814
 815         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 816         arvtool.formatgraph = None
 817
 818         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 819             j.run(runtimeContext)
 820
 821         _, kwargs = runner.api.container_requests().create.call_args
 822         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))