sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def helper(self, runner, enable_reuse=True):
  60         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  61
  62         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  63                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  64         loadingContext = arvados_cwl.context.ArvLoadingContext(
  65             {"avsc_names": avsc_names,
  66              "basedir": "",
  67              "make_fs_access": make_fs_access,
  68              "loader": Loader({}),
  69              "metadata": {"cwlVersion": "v1.0"}})
  70         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  71             {"work_api": "containers",
  72              "basedir": "",
  73              "name": "test_run_"+str(enable_reuse),
  74              "make_fs_access": make_fs_access,
  75              "tmpdir": "/tmp",
  76              "enable_reuse": enable_reuse,
  77              "priority": 500,
  78              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  79             })
  80
  81         return loadingContext, runtimeContext
  82
  83     # The test passes no builder.resources
  84     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  85     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  86     def test_run(self, keepdocker):
  87         for enable_reuse in (True, False):
  88             arv_docker_clear_cache()
  89
  90             runner = mock.MagicMock()
  91             runner.ignore_docker_for_reuse = False
  92             runner.intermediate_output_ttl = 0
  93             runner.secret_store = cwltool.secrets.SecretStore()
  94
  95             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  96             runner.api.collections().get().execute.return_value = {
  97                 "portable_data_hash": "99999999999999999999999999999993+99"}
  98
  99             tool = cmap({
 100                 "inputs": [],
 101                 "outputs": [],
 102                 "baseCommand": "ls",
 103                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 104                 "id": "#",
 105                 "class": "CommandLineTool"
 106             })
 107
 108             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 109
 110             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 111             arvtool.formatgraph = None
 112
 113             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 114                 j.run(runtimeContext)
 115                 runner.api.container_requests().create.assert_called_with(
 116                     body=JsonDiffMatcher({
 117                         'environment': {
 118                             'HOME': '/var/spool/cwl',
 119                             'TMPDIR': '/tmp'
 120                         },
 121                         'name': 'test_run_'+str(enable_reuse),
 122                         'runtime_constraints': {
 123                             'vcpus': 1,
 124                             'ram': 1073741824
 125                         },
 126                         'use_existing': enable_reuse,
 127                         'priority': 500,
 128                         'mounts': {
 129                             '/tmp': {'kind': 'tmp',
 130                                      "capacity": 1073741824
 131                                  },
 132                             '/var/spool/cwl': {'kind': 'tmp',
 133                                                "capacity": 1073741824 }
 134                         },
 135                         'state': 'Committed',
 136                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 137                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 138                         'output_path': '/var/spool/cwl',
 139                         'output_ttl': 0,
 140                         'container_image': '99999999999999999999999999999993+99',
 141                         'command': ['ls', '/var/spool/cwl'],
 142                         'cwd': '/var/spool/cwl',
 143                         'scheduling_parameters': {},
 144                         'properties': {},
 145                         'secret_mounts': {}
 146                     }))
 147
 148     # The test passes some fields in builder.resources
 149     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 150     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 151     def test_resource_requirements(self, keepdocker):
 152         arv_docker_clear_cache()
 153         runner = mock.MagicMock()
 154         runner.ignore_docker_for_reuse = False
 155         runner.intermediate_output_ttl = 3600
 156         runner.secret_store = cwltool.secrets.SecretStore()
 157
 158         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 159         runner.api.collections().get().execute.return_value = {
 160             "portable_data_hash": "99999999999999999999999999999993+99"}
 161
 162         tool = cmap({
 163             "inputs": [],
 164             "outputs": [],
 165             "hints": [{
 166                 "class": "ResourceRequirement",
 167                 "coresMin": 3,
 168                 "ramMin": 3000,
 169                 "tmpdirMin": 4000,
 170                 "outdirMin": 5000
 171             }, {
 172                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 173                 "keep_cache": 512
 174             }, {
 175                 "class": "http://arvados.org/cwl#APIRequirement",
 176             }, {
 177                 "class": "http://arvados.org/cwl#PartitionRequirement",
 178                 "partition": "blurb"
 179             }, {
 180                 "class": "http://arvados.org/cwl#IntermediateOutput",
 181                 "outputTTL": 7200
 182             }, {
 183                 "class": "http://arvados.org/cwl#ReuseRequirement",
 184                 "enableReuse": False
 185             }],
 186             "baseCommand": "ls",
 187             "id": "#",
 188             "class": "CommandLineTool"
 189         })
 190
 191         loadingContext, runtimeContext = self.helper(runner)
 192         runtimeContext.name = "test_resource_requirements"
 193
 194         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 195         arvtool.formatgraph = None
 196         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 197             j.run(runtimeContext)
 198
 199         call_args, call_kwargs = runner.api.container_requests().create.call_args
 200
 201         call_body_expected = {
 202             'environment': {
 203                 'HOME': '/var/spool/cwl',
 204                 'TMPDIR': '/tmp'
 205             },
 206             'name': 'test_resource_requirements',
 207             'runtime_constraints': {
 208                 'vcpus': 3,
 209                 'ram': 3145728000,
 210                 'keep_cache_ram': 536870912,
 211                 'API': True
 212             },
 213             'use_existing': False,
 214             'priority': 500,
 215             'mounts': {
 216                 '/tmp': {'kind': 'tmp',
 217                          "capacity": 4194304000 },
 218                 '/var/spool/cwl': {'kind': 'tmp',
 219                                    "capacity": 5242880000 }
 220             },
 221             'state': 'Committed',
 222             'output_name': 'Output for step test_resource_requirements',
 223             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 224             'output_path': '/var/spool/cwl',
 225             'output_ttl': 7200,
 226             'container_image': '99999999999999999999999999999993+99',
 227             'command': ['ls'],
 228             'cwd': '/var/spool/cwl',
 229             'scheduling_parameters': {
 230                 'partitions': ['blurb']
 231             },
 232             'properties': {},
 233             'secret_mounts': {}
 234         }
 235
 236         call_body = call_kwargs.get('body', None)
 237         self.assertNotEqual(None, call_body)
 238         for key in call_body:
 239             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 240
 241
 242     # The test passes some fields in builder.resources
 243     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 244     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 245     @mock.patch("arvados.collection.Collection")
 246     def test_initial_work_dir(self, collection_mock, keepdocker):
 247         arv_docker_clear_cache()
 248         runner = mock.MagicMock()
 249         runner.ignore_docker_for_reuse = False
 250         runner.intermediate_output_ttl = 0
 251         runner.secret_store = cwltool.secrets.SecretStore()
 252
 253         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 254         runner.api.collections().get().execute.return_value = {
 255             "portable_data_hash": "99999999999999999999999999999993+99"}
 256
 257         sourcemock = mock.MagicMock()
 258         def get_collection_mock(p):
 259             if "/" in p:
 260                 return (sourcemock, p.split("/", 1)[1])
 261             else:
 262                 return (sourcemock, "")
 263         runner.fs_access.get_collection.side_effect = get_collection_mock
 264
 265         vwdmock = mock.MagicMock()
 266         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 267
 268         tool = cmap({
 269             "inputs": [],
 270             "outputs": [],
 271             "hints": [{
 272                 "class": "InitialWorkDirRequirement",
 273                 "listing": [{
 274                     "class": "File",
 275                     "basename": "foo",
 276                     "location": "keep:99999999999999999999999999999995+99/bar"
 277                 },
 278                 {
 279                     "class": "Directory",
 280                     "basename": "foo2",
 281                     "location": "keep:99999999999999999999999999999995+99"
 282                 },
 283                 {
 284                     "class": "File",
 285                     "basename": "filename",
 286                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 287                 },
 288                 {
 289                     "class": "Directory",
 290                     "basename": "subdir",
 291                     "location": "keep:99999999999999999999999999999995+99/subdir"
 292                 }                        ]
 293             }],
 294             "baseCommand": "ls",
 295             "id": "#",
 296             "class": "CommandLineTool"
 297         })
 298
 299         loadingContext, runtimeContext = self.helper(runner)
 300         runtimeContext.name = "test_initial_work_dir"
 301
 302         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 303         arvtool.formatgraph = None
 304         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 305             j.run(runtimeContext)
 306
 307         call_args, call_kwargs = runner.api.container_requests().create.call_args
 308
 309         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 310         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 311         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 312         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 313
 314         call_body_expected = {
 315             'environment': {
 316                 'HOME': '/var/spool/cwl',
 317                 'TMPDIR': '/tmp'
 318             },
 319             'name': 'test_initial_work_dir',
 320             'runtime_constraints': {
 321                 'vcpus': 1,
 322                 'ram': 1073741824
 323             },
 324             'use_existing': True,
 325             'priority': 500,
 326             'mounts': {
 327                 '/tmp': {'kind': 'tmp',
 328                          "capacity": 1073741824 },
 329                 '/var/spool/cwl': {'kind': 'tmp',
 330                                    "capacity": 1073741824 },
 331                 '/var/spool/cwl/foo': {
 332                     'kind': 'collection',
 333                     'path': 'foo',
 334                     'portable_data_hash': '99999999999999999999999999999996+99'
 335                 },
 336                 '/var/spool/cwl/foo2': {
 337                     'kind': 'collection',
 338                     'path': 'foo2',
 339                     'portable_data_hash': '99999999999999999999999999999996+99'
 340                 },
 341                 '/var/spool/cwl/filename': {
 342                     'kind': 'collection',
 343                     'path': 'filename',
 344                     'portable_data_hash': '99999999999999999999999999999996+99'
 345                 },
 346                 '/var/spool/cwl/subdir': {
 347                     'kind': 'collection',
 348                     'path': 'subdir',
 349                     'portable_data_hash': '99999999999999999999999999999996+99'
 350                 }
 351             },
 352             'state': 'Committed',
 353             'output_name': 'Output for step test_initial_work_dir',
 354             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 355             'output_path': '/var/spool/cwl',
 356             'output_ttl': 0,
 357             'container_image': '99999999999999999999999999999993+99',
 358             'command': ['ls'],
 359             'cwd': '/var/spool/cwl',
 360             'scheduling_parameters': {
 361             },
 362             'properties': {},
 363             'secret_mounts': {}
 364         }
 365
 366         call_body = call_kwargs.get('body', None)
 367         self.assertNotEqual(None, call_body)
 368         for key in call_body:
 369             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 370
 371
 372     # Test redirecting stdin/stdout/stderr
 373     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 374     def test_redirects(self, keepdocker):
 375         arv_docker_clear_cache()
 376
 377         runner = mock.MagicMock()
 378         runner.ignore_docker_for_reuse = False
 379         runner.intermediate_output_ttl = 0
 380         runner.secret_store = cwltool.secrets.SecretStore()
 381
 382         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 383         runner.api.collections().get().execute.return_value = {
 384             "portable_data_hash": "99999999999999999999999999999993+99"}
 385
 386         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 387
 388         tool = cmap({
 389             "inputs": [],
 390             "outputs": [],
 391             "baseCommand": "ls",
 392             "stdout": "stdout.txt",
 393             "stderr": "stderr.txt",
 394             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 395             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 396             "id": "#",
 397             "class": "CommandLineTool"
 398         })
 399
 400         loadingContext, runtimeContext = self.helper(runner)
 401         runtimeContext.name = "test_run_redirect"
 402
 403         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 404         arvtool.formatgraph = None
 405         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 406             j.run(runtimeContext)
 407             runner.api.container_requests().create.assert_called_with(
 408                 body=JsonDiffMatcher({
 409                     'environment': {
 410                         'HOME': '/var/spool/cwl',
 411                         'TMPDIR': '/tmp'
 412                     },
 413                     'name': 'test_run_redirect',
 414                     'runtime_constraints': {
 415                         'vcpus': 1,
 416                         'ram': 1073741824
 417                     },
 418                     'use_existing': True,
 419                     'priority': 500,
 420                     'mounts': {
 421                         '/tmp': {'kind': 'tmp',
 422                                  "capacity": 1073741824 },
 423                         '/var/spool/cwl': {'kind': 'tmp',
 424                                            "capacity": 1073741824 },
 425                         "stderr": {
 426                             "kind": "file",
 427                             "path": "/var/spool/cwl/stderr.txt"
 428                         },
 429                         "stdin": {
 430                             "kind": "collection",
 431                             "path": "file.txt",
 432                             "portable_data_hash": "99999999999999999999999999999996+99"
 433                         },
 434                         "stdout": {
 435                             "kind": "file",
 436                             "path": "/var/spool/cwl/stdout.txt"
 437                         },
 438                     },
 439                     'state': 'Committed',
 440                     "output_name": "Output for step test_run_redirect",
 441                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 442                     'output_path': '/var/spool/cwl',
 443                     'output_ttl': 0,
 444                     'container_image': '99999999999999999999999999999993+99',
 445                     'command': ['ls', '/var/spool/cwl'],
 446                     'cwd': '/var/spool/cwl',
 447                     'scheduling_parameters': {},
 448                     'properties': {},
 449                     'secret_mounts': {}
 450                 }))
 451
 452     @mock.patch("arvados.collection.Collection")
 453     def test_done(self, col):
 454         api = mock.MagicMock()
 455
 456         runner = mock.MagicMock()
 457         runner.api = api
 458         runner.num_retries = 0
 459         runner.ignore_docker_for_reuse = False
 460         runner.intermediate_output_ttl = 0
 461         runner.secret_store = cwltool.secrets.SecretStore()
 462
 463         runner.api.containers().get().execute.return_value = {"state":"Complete",
 464                                                               "output": "abc+123",
 465                                                               "exit_code": 0}
 466
 467         col().open.return_value = []
 468
 469         loadingContext, runtimeContext = self.helper(runner)
 470
 471         arvjob = arvados_cwl.ArvadosContainer(runner,
 472                                               runtimeContext,
 473                                               mock.MagicMock(),
 474                                               {},
 475                                               None,
 476                                               [],
 477                                               [],
 478                                               "testjob")
 479         arvjob.output_callback = mock.MagicMock()
 480         arvjob.collect_outputs = mock.MagicMock()
 481         arvjob.successCodes = [0]
 482         arvjob.outdir = "/var/spool/cwl"
 483         arvjob.output_ttl = 3600
 484
 485         arvjob.collect_outputs.return_value = {"out": "stuff"}
 486
 487         arvjob.done({
 488             "state": "Final",
 489             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 490             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 491             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 492             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 493             "modified_at": "2017-05-26T12:01:22Z"
 494         })
 495
 496         self.assertFalse(api.collections().create.called)
 497         self.assertFalse(runner.runtime_status_error.called)
 498
 499         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 500         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 501         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 502
 503     # Test to make sure we dont call runtime_status_update if we already did
 504     # some where higher up in the call stack
 505     @mock.patch("arvados_cwl.util.get_current_container")
 506     def test_infinite_runtime_status_update(self, gcc_mock):
 507         api = mock.MagicMock()
 508         api._rootDesc = copy.deepcopy(get_rootDesc())
 509         del api._rootDesc.get('resources')['jobs']['methods']['create']
 510
 511         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 512         # adds the logging handler that will call runtime_status_update()
 513         runner = arvados_cwl.ArvCwlExecutor(api)
 514         self.assertEqual(runner.work_api, 'containers')
 515         root_logger = logging.getLogger('')
 516         handlerClasses = [h.__class__ for h in root_logger.handlers]
 517         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 518
 519         # get_current_container is invoked when we call runtime_status_update
 520         # so try and log again!
 521         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 522         try:
 523             root_logger.error("First Error")
 524         except RuntimeError as e:
 525             self.fail(str(e))
 526
 527     @mock.patch("arvados_cwl.util.get_current_container")
 528     @mock.patch("arvados.collection.CollectionReader")
 529     @mock.patch("arvados.collection.Collection")
 530     def test_child_failure(self, col, reader, gcc_mock):
 531         api = mock.MagicMock()
 532         api._rootDesc = copy.deepcopy(get_rootDesc())
 533         del api._rootDesc.get('resources')['jobs']['methods']['create']
 534
 535         # Set up runner with mocked runtime_status_update()
 536         self.assertFalse(gcc_mock.called)
 537         runtime_status_update = mock.MagicMock()
 538         arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update
 539         runner = arvados_cwl.ArvCwlExecutor(api)
 540         self.assertEqual(runner.work_api, 'containers')
 541
 542         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 543         # adds the logging handler that will call runtime_status_update() mock
 544         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 545         self.assertTrue(gcc_mock.called)
 546         root_logger = logging.getLogger('')
 547         handlerClasses = [h.__class__ for h in root_logger.handlers]
 548         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 549
 550         runner.num_retries = 0
 551         runner.ignore_docker_for_reuse = False
 552         runner.intermediate_output_ttl = 0
 553         runner.secret_store = cwltool.secrets.SecretStore()
 554         runner.label = mock.MagicMock()
 555         runner.label.return_value = '[container testjob]'
 556
 557         runner.api.containers().get().execute.return_value = {
 558             "state":"Complete",
 559             "output": "abc+123",
 560             "exit_code": 1,
 561             "log": "def+234"
 562         }
 563
 564         col().open.return_value = []
 565
 566         loadingContext, runtimeContext = self.helper(runner)
 567
 568         arvjob = arvados_cwl.ArvadosContainer(runner,
 569                                               runtimeContext,
 570                                               mock.MagicMock(),
 571                                               {},
 572                                               None,
 573                                               [],
 574                                               [],
 575                                               "testjob")
 576         arvjob.output_callback = mock.MagicMock()
 577         arvjob.collect_outputs = mock.MagicMock()
 578         arvjob.successCodes = [0]
 579         arvjob.outdir = "/var/spool/cwl"
 580         arvjob.output_ttl = 3600
 581         arvjob.collect_outputs.return_value = {"out": "stuff"}
 582
 583         arvjob.done({
 584             "state": "Final",
 585             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 586             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 587             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 588             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 589             "modified_at": "2017-05-26T12:01:22Z"
 590         })
 591
 592         runtime_status_update.assert_called_with(
 593             'error',
 594             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 595             '  ** log is empty **'
 596         )
 597         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 598
 599     # The test passes no builder.resources
 600     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 601     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 602     def test_mounts(self, keepdocker):
 603         arv_docker_clear_cache()
 604
 605         runner = mock.MagicMock()
 606         runner.ignore_docker_for_reuse = False
 607         runner.intermediate_output_ttl = 0
 608         runner.secret_store = cwltool.secrets.SecretStore()
 609
 610         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 611         runner.api.collections().get().execute.return_value = {
 612             "portable_data_hash": "99999999999999999999999999999994+99",
 613             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 614
 615         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 616
 617         tool = cmap({
 618             "inputs": [
 619                 {"id": "p1",
 620                  "type": "Directory"}
 621             ],
 622             "outputs": [],
 623             "baseCommand": "ls",
 624             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 625             "id": "#",
 626             "class": "CommandLineTool"
 627         })
 628
 629         loadingContext, runtimeContext = self.helper(runner)
 630         runtimeContext.name = "test_run_mounts"
 631
 632         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 633         arvtool.formatgraph = None
 634         job_order = {
 635             "p1": {
 636                 "class": "Directory",
 637                 "location": "keep:99999999999999999999999999999994+44",
 638                 "listing": [
 639                     {
 640                         "class": "File",
 641                         "location": "keep:99999999999999999999999999999994+44/file1",
 642                     },
 643                     {
 644                         "class": "File",
 645                         "location": "keep:99999999999999999999999999999994+44/file2",
 646                     }
 647                 ]
 648             }
 649         }
 650         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 651             j.run(runtimeContext)
 652             runner.api.container_requests().create.assert_called_with(
 653                 body=JsonDiffMatcher({
 654                     'environment': {
 655                         'HOME': '/var/spool/cwl',
 656                         'TMPDIR': '/tmp'
 657                     },
 658                     'name': 'test_run_mounts',
 659                     'runtime_constraints': {
 660                         'vcpus': 1,
 661                         'ram': 1073741824
 662                     },
 663                     'use_existing': True,
 664                     'priority': 500,
 665                     'mounts': {
 666                         "/keep/99999999999999999999999999999994+44": {
 667                             "kind": "collection",
 668                             "portable_data_hash": "99999999999999999999999999999994+44"
 669                         },
 670                         '/tmp': {'kind': 'tmp',
 671                                  "capacity": 1073741824 },
 672                         '/var/spool/cwl': {'kind': 'tmp',
 673                                            "capacity": 1073741824 }
 674                     },
 675                     'state': 'Committed',
 676                     'output_name': 'Output for step test_run_mounts',
 677                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 678                     'output_path': '/var/spool/cwl',
 679                     'output_ttl': 0,
 680                     'container_image': '99999999999999999999999999999994+99',
 681                     'command': ['ls', '/var/spool/cwl'],
 682                     'cwd': '/var/spool/cwl',
 683                     'scheduling_parameters': {},
 684                     'properties': {},
 685                     'secret_mounts': {}
 686                 }))
 687
 688     # The test passes no builder.resources
 689     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 690     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 691     def test_secrets(self, keepdocker):
 692         arv_docker_clear_cache()
 693
 694         runner = mock.MagicMock()
 695         runner.ignore_docker_for_reuse = False
 696         runner.intermediate_output_ttl = 0
 697         runner.secret_store = cwltool.secrets.SecretStore()
 698
 699         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 700         runner.api.collections().get().execute.return_value = {
 701             "portable_data_hash": "99999999999999999999999999999993+99"}
 702
 703         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 704
 705         tool = cmap({"arguments": ["md5sum", "example.conf"],
 706                      "class": "CommandLineTool",
 707                      "hints": [
 708                          {
 709                              "class": "http://commonwl.org/cwltool#Secrets",
 710                              "secrets": [
 711                                  "#secret_job.cwl/pw"
 712                              ]
 713                          }
 714                      ],
 715                      "id": "#secret_job.cwl",
 716                      "inputs": [
 717                          {
 718                              "id": "#secret_job.cwl/pw",
 719                              "type": "string"
 720                          }
 721                      ],
 722                      "outputs": [
 723                      ],
 724                      "requirements": [
 725                          {
 726                              "class": "InitialWorkDirRequirement",
 727                              "listing": [
 728                                  {
 729                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 730                                      "entryname": "example.conf"
 731                                  }
 732                              ]
 733                          }
 734                      ]})
 735
 736         loadingContext, runtimeContext = self.helper(runner)
 737         runtimeContext.name = "test_secrets"
 738
 739         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 740         arvtool.formatgraph = None
 741
 742         job_order = {"pw": "blorp"}
 743         runner.secret_store.store(["pw"], job_order)
 744
 745         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 746             j.run(runtimeContext)
 747             runner.api.container_requests().create.assert_called_with(
 748                 body=JsonDiffMatcher({
 749                     'environment': {
 750                         'HOME': '/var/spool/cwl',
 751                         'TMPDIR': '/tmp'
 752                     },
 753                     'name': 'test_secrets',
 754                     'runtime_constraints': {
 755                         'vcpus': 1,
 756                         'ram': 1073741824
 757                     },
 758                     'use_existing': True,
 759                     'priority': 500,
 760                     'mounts': {
 761                         '/tmp': {'kind': 'tmp',
 762                                  "capacity": 1073741824
 763                              },
 764                         '/var/spool/cwl': {'kind': 'tmp',
 765                                            "capacity": 1073741824 }
 766                     },
 767                     'state': 'Committed',
 768                     'output_name': 'Output for step test_secrets',
 769                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 770                     'output_path': '/var/spool/cwl',
 771                     'output_ttl': 0,
 772                     'container_image': '99999999999999999999999999999993+99',
 773                     'command': ['md5sum', 'example.conf'],
 774                     'cwd': '/var/spool/cwl',
 775                     'scheduling_parameters': {},
 776                     'properties': {},
 777                     "secret_mounts": {
 778                         "/var/spool/cwl/example.conf": {
 779                             "content": "username: user\npassword: blorp\n",
 780                             "kind": "text"
 781                         }
 782                     }
 783                 }))
 784
 785     # The test passes no builder.resources
 786     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 787     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 788     def test_timelimit(self, keepdocker):
 789         arv_docker_clear_cache()
 790
 791         runner = mock.MagicMock()
 792         runner.ignore_docker_for_reuse = False
 793         runner.intermediate_output_ttl = 0
 794         runner.secret_store = cwltool.secrets.SecretStore()
 795
 796         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 797         runner.api.collections().get().execute.return_value = {
 798             "portable_data_hash": "99999999999999999999999999999993+99"}
 799
 800         tool = cmap({
 801             "inputs": [],
 802             "outputs": [],
 803             "baseCommand": "ls",
 804             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 805             "id": "#",
 806             "class": "CommandLineTool",
 807             "hints": [
 808                 {
 809                     "class": "http://commonwl.org/cwltool#TimeLimit",
 810                     "timelimit": 42
 811                 }
 812             ]
 813         })
 814
 815         loadingContext, runtimeContext = self.helper(runner)
 816         runtimeContext.name = "test_timelimit"
 817
 818         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 819         arvtool.formatgraph = None
 820
 821         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 822             j.run(runtimeContext)
 823
 824         _, kwargs = runner.api.container_requests().create.call_args
 825         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))