sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher, StripYAMLComments
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def setUp(self):
  60         cwltool.process._names = set()
  61
  62     def helper(self, runner, enable_reuse=True):
  63         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
  64
  65         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  66                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  67         loadingContext = arvados_cwl.context.ArvLoadingContext(
  68             {"avsc_names": avsc_names,
  69              "basedir": "",
  70              "make_fs_access": make_fs_access,
  71              "loader": Loader({}),
  72              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  73         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  74             {"work_api": "containers",
  75              "basedir": "",
  76              "name": "test_run_"+str(enable_reuse),
  77              "make_fs_access": make_fs_access,
  78              "tmpdir": "/tmp",
  79              "enable_reuse": enable_reuse,
  80              "priority": 500,
  81              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82             })
  83
  84         return loadingContext, runtimeContext
  85
  86     # Helper function to set up the ArvCwlExecutor to use the containers api
  87     # and test that the RuntimeStatusLoggingHandler is set up correctly
  88     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  89         api = mock.MagicMock()
  90         api._rootDesc = copy.deepcopy(get_rootDesc())
  91         del api._rootDesc.get('resources')['jobs']['methods']['create']
  92
  93         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  94         # adds the logging handler that will call runtime_status_update() mock
  95         self.assertFalse(gcc_mock.called)
  96         runner = arvados_cwl.ArvCwlExecutor(api)
  97         self.assertEqual(runner.work_api, 'containers')
  98         root_logger = logging.getLogger('')
  99         handlerClasses = [h.__class__ for h in root_logger.handlers]
 100         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 101         return runner
 102
 103     # The test passes no builder.resources
 104     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 105     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 106     def test_run(self, keepdocker):
 107         for enable_reuse in (True, False):
 108             arv_docker_clear_cache()
 109
 110             runner = mock.MagicMock()
 111             runner.ignore_docker_for_reuse = False
 112             runner.intermediate_output_ttl = 0
 113             runner.secret_store = cwltool.secrets.SecretStore()
 114
 115             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 116             runner.api.collections().get().execute.return_value = {
 117                 "portable_data_hash": "99999999999999999999999999999993+99"}
 118
 119             tool = cmap({
 120                 "inputs": [],
 121                 "outputs": [],
 122                 "baseCommand": "ls",
 123                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 124                 "id": "#",
 125                 "class": "CommandLineTool"
 126             })
 127
 128             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 129
 130             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 131             arvtool.formatgraph = None
 132
 133             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 134                 j.run(runtimeContext)
 135                 runner.api.container_requests().create.assert_called_with(
 136                     body=JsonDiffMatcher({
 137                         'environment': {
 138                             'HOME': '/var/spool/cwl',
 139                             'TMPDIR': '/tmp'
 140                         },
 141                         'name': 'test_run_'+str(enable_reuse),
 142                         'runtime_constraints': {
 143                             'vcpus': 1,
 144                             'ram': 1073741824
 145                         },
 146                         'use_existing': enable_reuse,
 147                         'priority': 500,
 148                         'mounts': {
 149                             '/tmp': {'kind': 'tmp',
 150                                      "capacity": 1073741824
 151                                  },
 152                             '/var/spool/cwl': {'kind': 'tmp',
 153                                                "capacity": 1073741824 }
 154                         },
 155                         'state': 'Committed',
 156                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 157                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 158                         'output_path': '/var/spool/cwl',
 159                         'output_ttl': 0,
 160                         'container_image': '99999999999999999999999999999993+99',
 161                         'command': ['ls', '/var/spool/cwl'],
 162                         'cwd': '/var/spool/cwl',
 163                         'scheduling_parameters': {},
 164                         'properties': {},
 165                         'secret_mounts': {}
 166                     }))
 167
 168     # The test passes some fields in builder.resources
 169     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 170     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 171     def test_resource_requirements(self, keepdocker):
 172         arv_docker_clear_cache()
 173         runner = mock.MagicMock()
 174         runner.ignore_docker_for_reuse = False
 175         runner.intermediate_output_ttl = 3600
 176         runner.secret_store = cwltool.secrets.SecretStore()
 177
 178         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 179         runner.api.collections().get().execute.return_value = {
 180             "portable_data_hash": "99999999999999999999999999999993+99"}
 181
 182         tool = cmap({
 183             "inputs": [],
 184             "outputs": [],
 185             "hints": [{
 186                 "class": "ResourceRequirement",
 187                 "coresMin": 3,
 188                 "ramMin": 3000,
 189                 "tmpdirMin": 4000,
 190                 "outdirMin": 5000
 191             }, {
 192                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 193                 "keep_cache": 512
 194             }, {
 195                 "class": "http://arvados.org/cwl#APIRequirement",
 196             }, {
 197                 "class": "http://arvados.org/cwl#PartitionRequirement",
 198                 "partition": "blurb"
 199             }, {
 200                 "class": "http://arvados.org/cwl#IntermediateOutput",
 201                 "outputTTL": 7200
 202             }, {
 203                 "class": "http://arvados.org/cwl#ReuseRequirement",
 204                 "enableReuse": False
 205             }],
 206             "baseCommand": "ls",
 207             "id": "#",
 208             "class": "CommandLineTool"
 209         })
 210
 211         loadingContext, runtimeContext = self.helper(runner)
 212         runtimeContext.name = "test_resource_requirements"
 213
 214         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 215         arvtool.formatgraph = None
 216         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 217             j.run(runtimeContext)
 218
 219         call_args, call_kwargs = runner.api.container_requests().create.call_args
 220
 221         call_body_expected = {
 222             'environment': {
 223                 'HOME': '/var/spool/cwl',
 224                 'TMPDIR': '/tmp'
 225             },
 226             'name': 'test_resource_requirements',
 227             'runtime_constraints': {
 228                 'vcpus': 3,
 229                 'ram': 3145728000,
 230                 'keep_cache_ram': 536870912,
 231                 'API': True
 232             },
 233             'use_existing': False,
 234             'priority': 500,
 235             'mounts': {
 236                 '/tmp': {'kind': 'tmp',
 237                          "capacity": 4194304000 },
 238                 '/var/spool/cwl': {'kind': 'tmp',
 239                                    "capacity": 5242880000 }
 240             },
 241             'state': 'Committed',
 242             'output_name': 'Output for step test_resource_requirements',
 243             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 244             'output_path': '/var/spool/cwl',
 245             'output_ttl': 7200,
 246             'container_image': '99999999999999999999999999999993+99',
 247             'command': ['ls'],
 248             'cwd': '/var/spool/cwl',
 249             'scheduling_parameters': {
 250                 'partitions': ['blurb']
 251             },
 252             'properties': {},
 253             'secret_mounts': {}
 254         }
 255
 256         call_body = call_kwargs.get('body', None)
 257         self.assertNotEqual(None, call_body)
 258         for key in call_body:
 259             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 260
 261
 262     # The test passes some fields in builder.resources
 263     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 264     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 265     @mock.patch("arvados.collection.Collection")
 266     def test_initial_work_dir(self, collection_mock, keepdocker):
 267         arv_docker_clear_cache()
 268         runner = mock.MagicMock()
 269         runner.ignore_docker_for_reuse = False
 270         runner.intermediate_output_ttl = 0
 271         runner.secret_store = cwltool.secrets.SecretStore()
 272
 273         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 274         runner.api.collections().get().execute.return_value = {
 275             "portable_data_hash": "99999999999999999999999999999993+99"}
 276
 277         sourcemock = mock.MagicMock()
 278         def get_collection_mock(p):
 279             if "/" in p:
 280                 return (sourcemock, p.split("/", 1)[1])
 281             else:
 282                 return (sourcemock, "")
 283         runner.fs_access.get_collection.side_effect = get_collection_mock
 284
 285         vwdmock = mock.MagicMock()
 286         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 287
 288         tool = cmap({
 289             "inputs": [],
 290             "outputs": [],
 291             "hints": [{
 292                 "class": "InitialWorkDirRequirement",
 293                 "listing": [{
 294                     "class": "File",
 295                     "basename": "foo",
 296                     "location": "keep:99999999999999999999999999999995+99/bar"
 297                 },
 298                 {
 299                     "class": "Directory",
 300                     "basename": "foo2",
 301                     "location": "keep:99999999999999999999999999999995+99"
 302                 },
 303                 {
 304                     "class": "File",
 305                     "basename": "filename",
 306                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 307                 },
 308                 {
 309                     "class": "Directory",
 310                     "basename": "subdir",
 311                     "location": "keep:99999999999999999999999999999995+99/subdir"
 312                 }                        ]
 313             }],
 314             "baseCommand": "ls",
 315             "id": "#",
 316             "class": "CommandLineTool"
 317         })
 318
 319         loadingContext, runtimeContext = self.helper(runner)
 320         runtimeContext.name = "test_initial_work_dir"
 321
 322         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 323         arvtool.formatgraph = None
 324         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 325             j.run(runtimeContext)
 326
 327         call_args, call_kwargs = runner.api.container_requests().create.call_args
 328
 329         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 330         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 332         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 333
 334         call_body_expected = {
 335             'environment': {
 336                 'HOME': '/var/spool/cwl',
 337                 'TMPDIR': '/tmp'
 338             },
 339             'name': 'test_initial_work_dir',
 340             'runtime_constraints': {
 341                 'vcpus': 1,
 342                 'ram': 1073741824
 343             },
 344             'use_existing': True,
 345             'priority': 500,
 346             'mounts': {
 347                 '/tmp': {'kind': 'tmp',
 348                          "capacity": 1073741824 },
 349                 '/var/spool/cwl': {'kind': 'tmp',
 350                                    "capacity": 1073741824 },
 351                 '/var/spool/cwl/foo': {
 352                     'kind': 'collection',
 353                     'path': 'foo',
 354                     'portable_data_hash': '99999999999999999999999999999996+99'
 355                 },
 356                 '/var/spool/cwl/foo2': {
 357                     'kind': 'collection',
 358                     'path': 'foo2',
 359                     'portable_data_hash': '99999999999999999999999999999996+99'
 360                 },
 361                 '/var/spool/cwl/filename': {
 362                     'kind': 'collection',
 363                     'path': 'filename',
 364                     'portable_data_hash': '99999999999999999999999999999996+99'
 365                 },
 366                 '/var/spool/cwl/subdir': {
 367                     'kind': 'collection',
 368                     'path': 'subdir',
 369                     'portable_data_hash': '99999999999999999999999999999996+99'
 370                 }
 371             },
 372             'state': 'Committed',
 373             'output_name': 'Output for step test_initial_work_dir',
 374             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 375             'output_path': '/var/spool/cwl',
 376             'output_ttl': 0,
 377             'container_image': '99999999999999999999999999999993+99',
 378             'command': ['ls'],
 379             'cwd': '/var/spool/cwl',
 380             'scheduling_parameters': {
 381             },
 382             'properties': {},
 383             'secret_mounts': {}
 384         }
 385
 386         call_body = call_kwargs.get('body', None)
 387         self.assertNotEqual(None, call_body)
 388         for key in call_body:
 389             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 390
 391
 392     # Test redirecting stdin/stdout/stderr
 393     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 394     def test_redirects(self, keepdocker):
 395         arv_docker_clear_cache()
 396
 397         runner = mock.MagicMock()
 398         runner.ignore_docker_for_reuse = False
 399         runner.intermediate_output_ttl = 0
 400         runner.secret_store = cwltool.secrets.SecretStore()
 401
 402         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 403         runner.api.collections().get().execute.return_value = {
 404             "portable_data_hash": "99999999999999999999999999999993+99"}
 405
 406         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 407
 408         tool = cmap({
 409             "inputs": [],
 410             "outputs": [],
 411             "baseCommand": "ls",
 412             "stdout": "stdout.txt",
 413             "stderr": "stderr.txt",
 414             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 415             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 416             "id": "#",
 417             "class": "CommandLineTool"
 418         })
 419
 420         loadingContext, runtimeContext = self.helper(runner)
 421         runtimeContext.name = "test_run_redirect"
 422
 423         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 424         arvtool.formatgraph = None
 425         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 426             j.run(runtimeContext)
 427             runner.api.container_requests().create.assert_called_with(
 428                 body=JsonDiffMatcher({
 429                     'environment': {
 430                         'HOME': '/var/spool/cwl',
 431                         'TMPDIR': '/tmp'
 432                     },
 433                     'name': 'test_run_redirect',
 434                     'runtime_constraints': {
 435                         'vcpus': 1,
 436                         'ram': 1073741824
 437                     },
 438                     'use_existing': True,
 439                     'priority': 500,
 440                     'mounts': {
 441                         '/tmp': {'kind': 'tmp',
 442                                  "capacity": 1073741824 },
 443                         '/var/spool/cwl': {'kind': 'tmp',
 444                                            "capacity": 1073741824 },
 445                         "stderr": {
 446                             "kind": "file",
 447                             "path": "/var/spool/cwl/stderr.txt"
 448                         },
 449                         "stdin": {
 450                             "kind": "collection",
 451                             "path": "file.txt",
 452                             "portable_data_hash": "99999999999999999999999999999996+99"
 453                         },
 454                         "stdout": {
 455                             "kind": "file",
 456                             "path": "/var/spool/cwl/stdout.txt"
 457                         },
 458                     },
 459                     'state': 'Committed',
 460                     "output_name": "Output for step test_run_redirect",
 461                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 462                     'output_path': '/var/spool/cwl',
 463                     'output_ttl': 0,
 464                     'container_image': '99999999999999999999999999999993+99',
 465                     'command': ['ls', '/var/spool/cwl'],
 466                     'cwd': '/var/spool/cwl',
 467                     'scheduling_parameters': {},
 468                     'properties': {},
 469                     'secret_mounts': {}
 470                 }))
 471
 472     @mock.patch("arvados.collection.Collection")
 473     def test_done(self, col):
 474         api = mock.MagicMock()
 475
 476         runner = mock.MagicMock()
 477         runner.api = api
 478         runner.num_retries = 0
 479         runner.ignore_docker_for_reuse = False
 480         runner.intermediate_output_ttl = 0
 481         runner.secret_store = cwltool.secrets.SecretStore()
 482
 483         runner.api.containers().get().execute.return_value = {"state":"Complete",
 484                                                               "output": "abc+123",
 485                                                               "exit_code": 0}
 486
 487         col().open.return_value = []
 488
 489         loadingContext, runtimeContext = self.helper(runner)
 490
 491         arvjob = arvados_cwl.ArvadosContainer(runner,
 492                                               runtimeContext,
 493                                               mock.MagicMock(),
 494                                               {},
 495                                               None,
 496                                               [],
 497                                               [],
 498                                               "testjob")
 499         arvjob.output_callback = mock.MagicMock()
 500         arvjob.collect_outputs = mock.MagicMock()
 501         arvjob.successCodes = [0]
 502         arvjob.outdir = "/var/spool/cwl"
 503         arvjob.output_ttl = 3600
 504
 505         arvjob.collect_outputs.return_value = {"out": "stuff"}
 506
 507         arvjob.done({
 508             "state": "Final",
 509             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 510             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 511             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 512             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 513             "modified_at": "2017-05-26T12:01:22Z"
 514         })
 515
 516         self.assertFalse(api.collections().create.called)
 517         self.assertFalse(runner.runtime_status_error.called)
 518
 519         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 520         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 521         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 522
 523     # Test to make sure we dont call runtime_status_update if we already did
 524     # some where higher up in the call stack
 525     @mock.patch("arvados_cwl.util.get_current_container")
 526     def test_recursive_runtime_status_update(self, gcc_mock):
 527         self.setup_and_test_container_executor_and_logging(gcc_mock)
 528         root_logger = logging.getLogger('')
 529
 530         # get_current_container is invoked when we call runtime_status_update
 531         # so try and log again!
 532         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 533         try:
 534             root_logger.error("First Error")
 535         except RuntimeError:
 536             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 537
 538     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 539     @mock.patch("arvados_cwl.util.get_current_container")
 540     @mock.patch("arvados.collection.CollectionReader")
 541     @mock.patch("arvados.collection.Collection")
 542     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 543         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 544
 545         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 546         self.assertTrue(gcc_mock.called)
 547
 548         runner.num_retries = 0
 549         runner.ignore_docker_for_reuse = False
 550         runner.intermediate_output_ttl = 0
 551         runner.secret_store = cwltool.secrets.SecretStore()
 552         runner.label = mock.MagicMock()
 553         runner.label.return_value = '[container testjob]'
 554
 555         runner.api.containers().get().execute.return_value = {
 556             "state":"Complete",
 557             "output": "abc+123",
 558             "exit_code": 1,
 559             "log": "def+234"
 560         }
 561
 562         col().open.return_value = []
 563
 564         loadingContext, runtimeContext = self.helper(runner)
 565
 566         arvjob = arvados_cwl.ArvadosContainer(runner,
 567                                               runtimeContext,
 568                                               mock.MagicMock(),
 569                                               {},
 570                                               None,
 571                                               [],
 572                                               [],
 573                                               "testjob")
 574         arvjob.output_callback = mock.MagicMock()
 575         arvjob.collect_outputs = mock.MagicMock()
 576         arvjob.successCodes = [0]
 577         arvjob.outdir = "/var/spool/cwl"
 578         arvjob.output_ttl = 3600
 579         arvjob.collect_outputs.return_value = {"out": "stuff"}
 580
 581         arvjob.done({
 582             "state": "Final",
 583             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 584             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 585             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 586             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 587             "modified_at": "2017-05-26T12:01:22Z"
 588         })
 589
 590         rts_mock.assert_called_with(
 591             'error',
 592             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 593             '  ** log is empty **'
 594         )
 595         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 596
 597     # The test passes no builder.resources
 598     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 599     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 600     def test_mounts(self, keepdocker):
 601         arv_docker_clear_cache()
 602
 603         runner = mock.MagicMock()
 604         runner.ignore_docker_for_reuse = False
 605         runner.intermediate_output_ttl = 0
 606         runner.secret_store = cwltool.secrets.SecretStore()
 607
 608         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 609         runner.api.collections().get().execute.return_value = {
 610             "portable_data_hash": "99999999999999999999999999999994+99",
 611             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 612
 613         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 614
 615         tool = cmap({
 616             "inputs": [
 617                 {"id": "p1",
 618                  "type": "Directory"}
 619             ],
 620             "outputs": [],
 621             "baseCommand": "ls",
 622             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 623             "id": "#",
 624             "class": "CommandLineTool"
 625         })
 626
 627         loadingContext, runtimeContext = self.helper(runner)
 628         runtimeContext.name = "test_run_mounts"
 629
 630         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 631         arvtool.formatgraph = None
 632         job_order = {
 633             "p1": {
 634                 "class": "Directory",
 635                 "location": "keep:99999999999999999999999999999994+44",
 636                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 637                 "listing": [
 638                     {
 639                         "class": "File",
 640                         "location": "keep:99999999999999999999999999999994+44/file1",
 641                     },
 642                     {
 643                         "class": "File",
 644                         "location": "keep:99999999999999999999999999999994+44/file2",
 645                     }
 646                 ]
 647             }
 648         }
 649         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 650             j.run(runtimeContext)
 651             runner.api.container_requests().create.assert_called_with(
 652                 body=JsonDiffMatcher({
 653                     'environment': {
 654                         'HOME': '/var/spool/cwl',
 655                         'TMPDIR': '/tmp'
 656                     },
 657                     'name': 'test_run_mounts',
 658                     'runtime_constraints': {
 659                         'vcpus': 1,
 660                         'ram': 1073741824
 661                     },
 662                     'use_existing': True,
 663                     'priority': 500,
 664                     'mounts': {
 665                         "/keep/99999999999999999999999999999994+44": {
 666                             "kind": "collection",
 667                             "portable_data_hash": "99999999999999999999999999999994+44",
 668                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 669                         },
 670                         '/tmp': {'kind': 'tmp',
 671                                  "capacity": 1073741824 },
 672                         '/var/spool/cwl': {'kind': 'tmp',
 673                                            "capacity": 1073741824 }
 674                     },
 675                     'state': 'Committed',
 676                     'output_name': 'Output for step test_run_mounts',
 677                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 678                     'output_path': '/var/spool/cwl',
 679                     'output_ttl': 0,
 680                     'container_image': '99999999999999999999999999999994+99',
 681                     'command': ['ls', '/var/spool/cwl'],
 682                     'cwd': '/var/spool/cwl',
 683                     'scheduling_parameters': {},
 684                     'properties': {},
 685                     'secret_mounts': {}
 686                 }))
 687
 688     # The test passes no builder.resources
 689     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 690     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 691     def test_secrets(self, keepdocker):
 692         arv_docker_clear_cache()
 693
 694         runner = mock.MagicMock()
 695         runner.ignore_docker_for_reuse = False
 696         runner.intermediate_output_ttl = 0
 697         runner.secret_store = cwltool.secrets.SecretStore()
 698
 699         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 700         runner.api.collections().get().execute.return_value = {
 701             "portable_data_hash": "99999999999999999999999999999993+99"}
 702
 703         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 704
 705         tool = cmap({"arguments": ["md5sum", "example.conf"],
 706                      "class": "CommandLineTool",
 707                      "hints": [
 708                          {
 709                              "class": "http://commonwl.org/cwltool#Secrets",
 710                              "secrets": [
 711                                  "#secret_job.cwl/pw"
 712                              ]
 713                          }
 714                      ],
 715                      "id": "#secret_job.cwl",
 716                      "inputs": [
 717                          {
 718                              "id": "#secret_job.cwl/pw",
 719                              "type": "string"
 720                          }
 721                      ],
 722                      "outputs": [
 723                      ],
 724                      "requirements": [
 725                          {
 726                              "class": "InitialWorkDirRequirement",
 727                              "listing": [
 728                                  {
 729                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 730                                      "entryname": "example.conf"
 731                                  }
 732                              ]
 733                          }
 734                      ]})
 735
 736         loadingContext, runtimeContext = self.helper(runner)
 737         runtimeContext.name = "test_secrets"
 738
 739         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 740         arvtool.formatgraph = None
 741
 742         job_order = {"pw": "blorp"}
 743         runner.secret_store.store(["pw"], job_order)
 744
 745         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 746             j.run(runtimeContext)
 747             runner.api.container_requests().create.assert_called_with(
 748                 body=JsonDiffMatcher({
 749                     'environment': {
 750                         'HOME': '/var/spool/cwl',
 751                         'TMPDIR': '/tmp'
 752                     },
 753                     'name': 'test_secrets',
 754                     'runtime_constraints': {
 755                         'vcpus': 1,
 756                         'ram': 1073741824
 757                     },
 758                     'use_existing': True,
 759                     'priority': 500,
 760                     'mounts': {
 761                         '/tmp': {'kind': 'tmp',
 762                                  "capacity": 1073741824
 763                              },
 764                         '/var/spool/cwl': {'kind': 'tmp',
 765                                            "capacity": 1073741824 }
 766                     },
 767                     'state': 'Committed',
 768                     'output_name': 'Output for step test_secrets',
 769                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 770                     'output_path': '/var/spool/cwl',
 771                     'output_ttl': 0,
 772                     'container_image': '99999999999999999999999999999993+99',
 773                     'command': ['md5sum', 'example.conf'],
 774                     'cwd': '/var/spool/cwl',
 775                     'scheduling_parameters': {},
 776                     'properties': {},
 777                     "secret_mounts": {
 778                         "/var/spool/cwl/example.conf": {
 779                             "content": "username: user\npassword: blorp\n",
 780                             "kind": "text"
 781                         }
 782                     }
 783                 }))
 784
 785     # The test passes no builder.resources
 786     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 787     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 788     def test_timelimit(self, keepdocker):
 789         arv_docker_clear_cache()
 790
 791         runner = mock.MagicMock()
 792         runner.ignore_docker_for_reuse = False
 793         runner.intermediate_output_ttl = 0
 794         runner.secret_store = cwltool.secrets.SecretStore()
 795
 796         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 797         runner.api.collections().get().execute.return_value = {
 798             "portable_data_hash": "99999999999999999999999999999993+99"}
 799
 800         tool = cmap({
 801             "inputs": [],
 802             "outputs": [],
 803             "baseCommand": "ls",
 804             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 805             "id": "#",
 806             "class": "CommandLineTool",
 807             "hints": [
 808                 {
 809                     "class": "ToolTimeLimit",
 810                     "timelimit": 42
 811                 }
 812             ]
 813         })
 814
 815         loadingContext, runtimeContext = self.helper(runner)
 816         runtimeContext.name = "test_timelimit"
 817
 818         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 819         arvtool.formatgraph = None
 820
 821         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 822             j.run(runtimeContext)
 823
 824         _, kwargs = runner.api.container_requests().create.call_args
 825         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 826
 827
 828 class TestWorkflow(unittest.TestCase):
 829     def setUp(self):
 830         cwltool.process._names = set()
 831
 832     def helper(self, runner, enable_reuse=True):
 833         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 834
 835         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 836                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 837
 838         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 839         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 840         document_loader.fetch_text = document_loader.fetcher.fetch_text
 841         document_loader.check_exists = document_loader.fetcher.check_exists
 842
 843         loadingContext = arvados_cwl.context.ArvLoadingContext(
 844             {"avsc_names": avsc_names,
 845              "basedir": "",
 846              "make_fs_access": make_fs_access,
 847              "loader": document_loader,
 848              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 849              "construct_tool_object": runner.arv_make_tool})
 850         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 851             {"work_api": "containers",
 852              "basedir": "",
 853              "name": "test_run_wf_"+str(enable_reuse),
 854              "make_fs_access": make_fs_access,
 855              "tmpdir": "/tmp",
 856              "enable_reuse": enable_reuse,
 857              "priority": 500})
 858
 859         return loadingContext, runtimeContext
 860
 861     # The test passes no builder.resources
 862     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 863     @mock.patch("arvados.collection.CollectionReader")
 864     @mock.patch("arvados.collection.Collection")
 865     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 866     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 867         arv_docker_clear_cache()
 868         arvados_cwl.add_arv_hints()
 869
 870         api = mock.MagicMock()
 871         api._rootDesc = get_rootDesc()
 872
 873         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 874         self.assertEqual(runner.work_api, 'containers')
 875
 876         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 877         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 878         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 879                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 880
 881         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 882         runner.ignore_docker_for_reuse = False
 883         runner.num_retries = 0
 884         runner.secret_store = cwltool.secrets.SecretStore()
 885
 886         loadingContext, runtimeContext = self.helper(runner)
 887         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 888
 889         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 890         metadata["cwlVersion"] = tool["cwlVersion"]
 891
 892         mockc = mock.MagicMock()
 893         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 894         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 895
 896         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 897         arvtool.formatgraph = None
 898         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 899
 900         next(it).run(runtimeContext)
 901         next(it).run(runtimeContext)
 902
 903         with open("tests/wf/scatter2_subwf.cwl") as f:
 904             subwf = StripYAMLComments(f.read()).rstrip()
 905
 906         runner.api.container_requests().create.assert_called_with(
 907             body=JsonDiffMatcher({
 908                 "command": [
 909                     "cwltool",
 910                     "--no-container",
 911                     "--move-outputs",
 912                     "--preserve-entire-environment",
 913                     "workflow.cwl#main",
 914                     "cwl.input.yml"
 915                 ],
 916                 "container_image": "99999999999999999999999999999993+99",
 917                 "cwd": "/var/spool/cwl",
 918                 "environment": {
 919                     "HOME": "/var/spool/cwl",
 920                     "TMPDIR": "/tmp"
 921                 },
 922                 "mounts": {
 923                     "/keep/99999999999999999999999999999999+118": {
 924                         "kind": "collection",
 925                         "portable_data_hash": "99999999999999999999999999999999+118"
 926                     },
 927                     "/tmp": {
 928                         "capacity": 1073741824,
 929                         "kind": "tmp"
 930                     },
 931                     "/var/spool/cwl": {
 932                         "capacity": 1073741824,
 933                         "kind": "tmp"
 934                     },
 935                     "/var/spool/cwl/cwl.input.yml": {
 936                         "kind": "collection",
 937                         "path": "cwl.input.yml",
 938                         "portable_data_hash": "99999999999999999999999999999996+99"
 939                     },
 940                     "/var/spool/cwl/workflow.cwl": {
 941                         "kind": "collection",
 942                         "path": "workflow.cwl",
 943                         "portable_data_hash": "99999999999999999999999999999996+99"
 944                     },
 945                     "stdout": {
 946                         "kind": "file",
 947                         "path": "/var/spool/cwl/cwl.output.json"
 948                     }
 949                 },
 950                 "name": "scatterstep",
 951                 "output_name": "Output for step scatterstep",
 952                 "output_path": "/var/spool/cwl",
 953                 "output_ttl": 0,
 954                 "priority": 500,
 955                 "properties": {},
 956                 "runtime_constraints": {
 957                     "ram": 1073741824,
 958                     "vcpus": 1
 959                 },
 960                 "scheduling_parameters": {},
 961                 "secret_mounts": {},
 962                 "state": "Committed",
 963                 "use_existing": True
 964             }))
 965         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 966         mockc.open().__enter__().write.assert_has_calls([mock.call(
 967 '''{
 968   "fileblub": {
 969     "basename": "token.txt",
 970     "class": "File",
 971     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 972     "size": 0
 973   },
 974   "sleeptime": 5
 975 }''')])
 976
 977     # The test passes no builder.resources
 978     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 979     @mock.patch("arvados.collection.CollectionReader")
 980     @mock.patch("arvados.collection.Collection")
 981     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 982     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 983         arv_docker_clear_cache()
 984         arvados_cwl.add_arv_hints()
 985
 986         api = mock.MagicMock()
 987         api._rootDesc = get_rootDesc()
 988
 989         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 990         self.assertEqual(runner.work_api, 'containers')
 991
 992         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 993         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 994                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
 995         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 996                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 997
 998         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 999         runner.ignore_docker_for_reuse = False
1000         runner.num_retries = 0
1001         runner.secret_store = cwltool.secrets.SecretStore()
1002
1003         loadingContext, runtimeContext = self.helper(runner)
1004         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1005         loadingContext.do_update = True
1006         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1007
1008         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1009
1010         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1011         arvtool.formatgraph = None
1012         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1013
1014         next(it).run(runtimeContext)
1015         next(it).run(runtimeContext)
1016
1017         with open("tests/wf/echo-subwf.cwl") as f:
1018             subwf = StripYAMLComments(f.read())
1019
1020         runner.api.container_requests().create.assert_called_with(
1021             body=JsonDiffMatcher({
1022                 'output_ttl': 0,
1023                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1024                 'scheduling_parameters': {},
1025                 'name': u'echo-subwf',
1026                 'secret_mounts': {},
1027                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1028                 'properties': {},
1029                 'priority': 500,
1030                 'mounts': {
1031                     '/var/spool/cwl/cwl.input.yml': {
1032                         'portable_data_hash': '99999999999999999999999999999996+99',
1033                         'kind': 'collection',
1034                         'path': 'cwl.input.yml'
1035                     },
1036                     '/var/spool/cwl/workflow.cwl': {
1037                         'portable_data_hash': '99999999999999999999999999999996+99',
1038                         'kind': 'collection',
1039                         'path': 'workflow.cwl'
1040                     },
1041                     'stdout': {
1042                         'path': '/var/spool/cwl/cwl.output.json',
1043                         'kind': 'file'
1044                     },
1045                     '/tmp': {
1046                         'kind': 'tmp',
1047                         'capacity': 1073741824
1048                     }, '/var/spool/cwl': {
1049                         'kind': 'tmp',
1050                         'capacity': 3221225472
1051                     }
1052                 },
1053                 'state': 'Committed',
1054                 'output_path': '/var/spool/cwl',
1055                 'container_image': '99999999999999999999999999999993+99',
1056                 'command': [
1057                     u'cwltool',
1058                     u'--no-container',
1059                     u'--move-outputs',
1060                     u'--preserve-entire-environment',
1061                     u'workflow.cwl#main',
1062                     u'cwl.input.yml'
1063                 ],
1064                 'use_existing': True,
1065                 'output_name': u'Output for step echo-subwf',
1066                 'cwd': '/var/spool/cwl'
1067             }))
1068
1069     def test_default_work_api(self):
1070         arvados_cwl.add_arv_hints()
1071
1072         api = mock.MagicMock()
1073         api._rootDesc = copy.deepcopy(get_rootDesc())
1074         del api._rootDesc.get('resources')['jobs']['methods']['create']
1075         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1076         self.assertEqual(runner.work_api, 'containers')