sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher, StripYAMLComments
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def setUp(self):
  60         cwltool.process._names = set()
  61
  62     def helper(self, runner, enable_reuse=True):
  63         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
  64
  65         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  66                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  67         loadingContext = arvados_cwl.context.ArvLoadingContext(
  68             {"avsc_names": avsc_names,
  69              "basedir": "",
  70              "make_fs_access": make_fs_access,
  71              "loader": Loader({}),
  72              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  73         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  74             {"work_api": "containers",
  75              "basedir": "",
  76              "name": "test_run_"+str(enable_reuse),
  77              "make_fs_access": make_fs_access,
  78              "tmpdir": "/tmp",
  79              "enable_reuse": enable_reuse,
  80              "priority": 500,
  81              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82             })
  83
  84         return loadingContext, runtimeContext
  85
  86     # Helper function to set up the ArvCwlExecutor to use the containers api
  87     # and test that the RuntimeStatusLoggingHandler is set up correctly
  88     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  89         api = mock.MagicMock()
  90         api._rootDesc = copy.deepcopy(get_rootDesc())
  91
  92         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  93         # adds the logging handler that will call runtime_status_update() mock
  94         self.assertFalse(gcc_mock.called)
  95         runner = arvados_cwl.ArvCwlExecutor(api)
  96         self.assertEqual(runner.work_api, 'containers')
  97         root_logger = logging.getLogger('')
  98         handlerClasses = [h.__class__ for h in root_logger.handlers]
  99         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 100         return runner
 101
 102     # The test passes no builder.resources
 103     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 104     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 105     def test_run(self, keepdocker):
 106         for enable_reuse in (True, False):
 107             arv_docker_clear_cache()
 108
 109             runner = mock.MagicMock()
 110             runner.ignore_docker_for_reuse = False
 111             runner.intermediate_output_ttl = 0
 112             runner.secret_store = cwltool.secrets.SecretStore()
 113
 114             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 115             runner.api.collections().get().execute.return_value = {
 116                 "portable_data_hash": "99999999999999999999999999999993+99"}
 117
 118             tool = cmap({
 119                 "inputs": [],
 120                 "outputs": [],
 121                 "baseCommand": "ls",
 122                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 123                 "id": "#",
 124                 "class": "CommandLineTool"
 125             })
 126
 127             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 128
 129             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 130             arvtool.formatgraph = None
 131
 132             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 133                 j.run(runtimeContext)
 134                 runner.api.container_requests().create.assert_called_with(
 135                     body=JsonDiffMatcher({
 136                         'environment': {
 137                             'HOME': '/var/spool/cwl',
 138                             'TMPDIR': '/tmp'
 139                         },
 140                         'name': 'test_run_'+str(enable_reuse),
 141                         'runtime_constraints': {
 142                             'vcpus': 1,
 143                             'ram': 1073741824
 144                         },
 145                         'use_existing': enable_reuse,
 146                         'priority': 500,
 147                         'mounts': {
 148                             '/tmp': {'kind': 'tmp',
 149                                      "capacity": 1073741824
 150                                  },
 151                             '/var/spool/cwl': {'kind': 'tmp',
 152                                                "capacity": 1073741824 }
 153                         },
 154                         'state': 'Committed',
 155                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 156                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 157                         'output_path': '/var/spool/cwl',
 158                         'output_ttl': 0,
 159                         'container_image': '99999999999999999999999999999993+99',
 160                         'command': ['ls', '/var/spool/cwl'],
 161                         'cwd': '/var/spool/cwl',
 162                         'scheduling_parameters': {},
 163                         'properties': {},
 164                         'secret_mounts': {}
 165                     }))
 166
 167     # The test passes some fields in builder.resources
 168     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 169     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 170     def test_resource_requirements(self, keepdocker):
 171         arv_docker_clear_cache()
 172         runner = mock.MagicMock()
 173         runner.ignore_docker_for_reuse = False
 174         runner.intermediate_output_ttl = 3600
 175         runner.secret_store = cwltool.secrets.SecretStore()
 176
 177         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 178         runner.api.collections().get().execute.return_value = {
 179             "portable_data_hash": "99999999999999999999999999999993+99"}
 180
 181         tool = cmap({
 182             "inputs": [],
 183             "outputs": [],
 184             "hints": [{
 185                 "class": "ResourceRequirement",
 186                 "coresMin": 3,
 187                 "ramMin": 3000,
 188                 "tmpdirMin": 4000,
 189                 "outdirMin": 5000
 190             }, {
 191                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 192                 "keep_cache": 512
 193             }, {
 194                 "class": "http://arvados.org/cwl#APIRequirement",
 195             }, {
 196                 "class": "http://arvados.org/cwl#PartitionRequirement",
 197                 "partition": "blurb"
 198             }, {
 199                 "class": "http://arvados.org/cwl#IntermediateOutput",
 200                 "outputTTL": 7200
 201             }, {
 202                 "class": "http://arvados.org/cwl#ReuseRequirement",
 203                 "enableReuse": False
 204             }],
 205             "baseCommand": "ls",
 206             "id": "#",
 207             "class": "CommandLineTool"
 208         })
 209
 210         loadingContext, runtimeContext = self.helper(runner)
 211         runtimeContext.name = "test_resource_requirements"
 212
 213         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 214         arvtool.formatgraph = None
 215         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 216             j.run(runtimeContext)
 217
 218         call_args, call_kwargs = runner.api.container_requests().create.call_args
 219
 220         call_body_expected = {
 221             'environment': {
 222                 'HOME': '/var/spool/cwl',
 223                 'TMPDIR': '/tmp'
 224             },
 225             'name': 'test_resource_requirements',
 226             'runtime_constraints': {
 227                 'vcpus': 3,
 228                 'ram': 3145728000,
 229                 'keep_cache_ram': 536870912,
 230                 'API': True
 231             },
 232             'use_existing': False,
 233             'priority': 500,
 234             'mounts': {
 235                 '/tmp': {'kind': 'tmp',
 236                          "capacity": 4194304000 },
 237                 '/var/spool/cwl': {'kind': 'tmp',
 238                                    "capacity": 5242880000 }
 239             },
 240             'state': 'Committed',
 241             'output_name': 'Output for step test_resource_requirements',
 242             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 243             'output_path': '/var/spool/cwl',
 244             'output_ttl': 7200,
 245             'container_image': '99999999999999999999999999999993+99',
 246             'command': ['ls'],
 247             'cwd': '/var/spool/cwl',
 248             'scheduling_parameters': {
 249                 'partitions': ['blurb']
 250             },
 251             'properties': {},
 252             'secret_mounts': {}
 253         }
 254
 255         call_body = call_kwargs.get('body', None)
 256         self.assertNotEqual(None, call_body)
 257         for key in call_body:
 258             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 259
 260
 261     # The test passes some fields in builder.resources
 262     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 263     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 264     @mock.patch("arvados.collection.Collection")
 265     def test_initial_work_dir(self, collection_mock, keepdocker):
 266         arv_docker_clear_cache()
 267         runner = mock.MagicMock()
 268         runner.ignore_docker_for_reuse = False
 269         runner.intermediate_output_ttl = 0
 270         runner.secret_store = cwltool.secrets.SecretStore()
 271
 272         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 273         runner.api.collections().get().execute.return_value = {
 274             "portable_data_hash": "99999999999999999999999999999993+99"}
 275
 276         sourcemock = mock.MagicMock()
 277         def get_collection_mock(p):
 278             if "/" in p:
 279                 return (sourcemock, p.split("/", 1)[1])
 280             else:
 281                 return (sourcemock, "")
 282         runner.fs_access.get_collection.side_effect = get_collection_mock
 283
 284         vwdmock = mock.MagicMock()
 285         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 286
 287         tool = cmap({
 288             "inputs": [],
 289             "outputs": [],
 290             "hints": [{
 291                 "class": "InitialWorkDirRequirement",
 292                 "listing": [{
 293                     "class": "File",
 294                     "basename": "foo",
 295                     "location": "keep:99999999999999999999999999999995+99/bar"
 296                 },
 297                 {
 298                     "class": "Directory",
 299                     "basename": "foo2",
 300                     "location": "keep:99999999999999999999999999999995+99"
 301                 },
 302                 {
 303                     "class": "File",
 304                     "basename": "filename",
 305                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 306                 },
 307                 {
 308                     "class": "Directory",
 309                     "basename": "subdir",
 310                     "location": "keep:99999999999999999999999999999995+99/subdir"
 311                 }                        ]
 312             }],
 313             "baseCommand": "ls",
 314             "id": "#",
 315             "class": "CommandLineTool"
 316         })
 317
 318         loadingContext, runtimeContext = self.helper(runner)
 319         runtimeContext.name = "test_initial_work_dir"
 320
 321         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 322         arvtool.formatgraph = None
 323         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 324             j.run(runtimeContext)
 325
 326         call_args, call_kwargs = runner.api.container_requests().create.call_args
 327
 328         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 329         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 330         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 332
 333         call_body_expected = {
 334             'environment': {
 335                 'HOME': '/var/spool/cwl',
 336                 'TMPDIR': '/tmp'
 337             },
 338             'name': 'test_initial_work_dir',
 339             'runtime_constraints': {
 340                 'vcpus': 1,
 341                 'ram': 1073741824
 342             },
 343             'use_existing': True,
 344             'priority': 500,
 345             'mounts': {
 346                 '/tmp': {'kind': 'tmp',
 347                          "capacity": 1073741824 },
 348                 '/var/spool/cwl': {'kind': 'tmp',
 349                                    "capacity": 1073741824 },
 350                 '/var/spool/cwl/foo': {
 351                     'kind': 'collection',
 352                     'path': 'foo',
 353                     'portable_data_hash': '99999999999999999999999999999996+99'
 354                 },
 355                 '/var/spool/cwl/foo2': {
 356                     'kind': 'collection',
 357                     'path': 'foo2',
 358                     'portable_data_hash': '99999999999999999999999999999996+99'
 359                 },
 360                 '/var/spool/cwl/filename': {
 361                     'kind': 'collection',
 362                     'path': 'filename',
 363                     'portable_data_hash': '99999999999999999999999999999996+99'
 364                 },
 365                 '/var/spool/cwl/subdir': {
 366                     'kind': 'collection',
 367                     'path': 'subdir',
 368                     'portable_data_hash': '99999999999999999999999999999996+99'
 369                 }
 370             },
 371             'state': 'Committed',
 372             'output_name': 'Output for step test_initial_work_dir',
 373             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 374             'output_path': '/var/spool/cwl',
 375             'output_ttl': 0,
 376             'container_image': '99999999999999999999999999999993+99',
 377             'command': ['ls'],
 378             'cwd': '/var/spool/cwl',
 379             'scheduling_parameters': {
 380             },
 381             'properties': {},
 382             'secret_mounts': {}
 383         }
 384
 385         call_body = call_kwargs.get('body', None)
 386         self.assertNotEqual(None, call_body)
 387         for key in call_body:
 388             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 389
 390
 391     # Test redirecting stdin/stdout/stderr
 392     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 393     def test_redirects(self, keepdocker):
 394         arv_docker_clear_cache()
 395
 396         runner = mock.MagicMock()
 397         runner.ignore_docker_for_reuse = False
 398         runner.intermediate_output_ttl = 0
 399         runner.secret_store = cwltool.secrets.SecretStore()
 400
 401         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 402         runner.api.collections().get().execute.return_value = {
 403             "portable_data_hash": "99999999999999999999999999999993+99"}
 404
 405         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 406
 407         tool = cmap({
 408             "inputs": [],
 409             "outputs": [],
 410             "baseCommand": "ls",
 411             "stdout": "stdout.txt",
 412             "stderr": "stderr.txt",
 413             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 414             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 415             "id": "#",
 416             "class": "CommandLineTool"
 417         })
 418
 419         loadingContext, runtimeContext = self.helper(runner)
 420         runtimeContext.name = "test_run_redirect"
 421
 422         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 423         arvtool.formatgraph = None
 424         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 425             j.run(runtimeContext)
 426             runner.api.container_requests().create.assert_called_with(
 427                 body=JsonDiffMatcher({
 428                     'environment': {
 429                         'HOME': '/var/spool/cwl',
 430                         'TMPDIR': '/tmp'
 431                     },
 432                     'name': 'test_run_redirect',
 433                     'runtime_constraints': {
 434                         'vcpus': 1,
 435                         'ram': 1073741824
 436                     },
 437                     'use_existing': True,
 438                     'priority': 500,
 439                     'mounts': {
 440                         '/tmp': {'kind': 'tmp',
 441                                  "capacity": 1073741824 },
 442                         '/var/spool/cwl': {'kind': 'tmp',
 443                                            "capacity": 1073741824 },
 444                         "stderr": {
 445                             "kind": "file",
 446                             "path": "/var/spool/cwl/stderr.txt"
 447                         },
 448                         "stdin": {
 449                             "kind": "collection",
 450                             "path": "file.txt",
 451                             "portable_data_hash": "99999999999999999999999999999996+99"
 452                         },
 453                         "stdout": {
 454                             "kind": "file",
 455                             "path": "/var/spool/cwl/stdout.txt"
 456                         },
 457                     },
 458                     'state': 'Committed',
 459                     "output_name": "Output for step test_run_redirect",
 460                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 461                     'output_path': '/var/spool/cwl',
 462                     'output_ttl': 0,
 463                     'container_image': '99999999999999999999999999999993+99',
 464                     'command': ['ls', '/var/spool/cwl'],
 465                     'cwd': '/var/spool/cwl',
 466                     'scheduling_parameters': {},
 467                     'properties': {},
 468                     'secret_mounts': {}
 469                 }))
 470
 471     @mock.patch("arvados.collection.Collection")
 472     def test_done(self, col):
 473         api = mock.MagicMock()
 474
 475         runner = mock.MagicMock()
 476         runner.api = api
 477         runner.num_retries = 0
 478         runner.ignore_docker_for_reuse = False
 479         runner.intermediate_output_ttl = 0
 480         runner.secret_store = cwltool.secrets.SecretStore()
 481
 482         runner.api.containers().get().execute.return_value = {"state":"Complete",
 483                                                               "output": "abc+123",
 484                                                               "exit_code": 0}
 485
 486         col().open.return_value = []
 487
 488         loadingContext, runtimeContext = self.helper(runner)
 489
 490         arvjob = arvados_cwl.ArvadosContainer(runner,
 491                                               runtimeContext,
 492                                               mock.MagicMock(),
 493                                               {},
 494                                               None,
 495                                               [],
 496                                               [],
 497                                               "testjob")
 498         arvjob.output_callback = mock.MagicMock()
 499         arvjob.collect_outputs = mock.MagicMock()
 500         arvjob.successCodes = [0]
 501         arvjob.outdir = "/var/spool/cwl"
 502         arvjob.output_ttl = 3600
 503
 504         arvjob.collect_outputs.return_value = {"out": "stuff"}
 505
 506         arvjob.done({
 507             "state": "Final",
 508             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 509             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 510             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 511             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 512             "modified_at": "2017-05-26T12:01:22Z"
 513         })
 514
 515         self.assertFalse(api.collections().create.called)
 516         self.assertFalse(runner.runtime_status_error.called)
 517
 518         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 519         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 520         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 521
 522     # Test to make sure we dont call runtime_status_update if we already did
 523     # some where higher up in the call stack
 524     @mock.patch("arvados_cwl.util.get_current_container")
 525     def test_recursive_runtime_status_update(self, gcc_mock):
 526         self.setup_and_test_container_executor_and_logging(gcc_mock)
 527         root_logger = logging.getLogger('')
 528
 529         # get_current_container is invoked when we call runtime_status_update
 530         # so try and log again!
 531         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 532         try:
 533             root_logger.error("First Error")
 534         except RuntimeError:
 535             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 536
 537     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 538     @mock.patch("arvados_cwl.util.get_current_container")
 539     @mock.patch("arvados.collection.CollectionReader")
 540     @mock.patch("arvados.collection.Collection")
 541     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 542         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 543
 544         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 545         self.assertTrue(gcc_mock.called)
 546
 547         runner.num_retries = 0
 548         runner.ignore_docker_for_reuse = False
 549         runner.intermediate_output_ttl = 0
 550         runner.secret_store = cwltool.secrets.SecretStore()
 551         runner.label = mock.MagicMock()
 552         runner.label.return_value = '[container testjob]'
 553
 554         runner.api.containers().get().execute.return_value = {
 555             "state":"Complete",
 556             "output": "abc+123",
 557             "exit_code": 1,
 558             "log": "def+234"
 559         }
 560
 561         col().open.return_value = []
 562
 563         loadingContext, runtimeContext = self.helper(runner)
 564
 565         arvjob = arvados_cwl.ArvadosContainer(runner,
 566                                               runtimeContext,
 567                                               mock.MagicMock(),
 568                                               {},
 569                                               None,
 570                                               [],
 571                                               [],
 572                                               "testjob")
 573         arvjob.output_callback = mock.MagicMock()
 574         arvjob.collect_outputs = mock.MagicMock()
 575         arvjob.successCodes = [0]
 576         arvjob.outdir = "/var/spool/cwl"
 577         arvjob.output_ttl = 3600
 578         arvjob.collect_outputs.return_value = {"out": "stuff"}
 579
 580         arvjob.done({
 581             "state": "Final",
 582             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 583             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 584             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 585             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 586             "modified_at": "2017-05-26T12:01:22Z"
 587         })
 588
 589         rts_mock.assert_called_with(
 590             'error',
 591             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 592             '  ** log is empty **'
 593         )
 594         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 595
 596     # The test passes no builder.resources
 597     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 598     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 599     def test_mounts(self, keepdocker):
 600         arv_docker_clear_cache()
 601
 602         runner = mock.MagicMock()
 603         runner.ignore_docker_for_reuse = False
 604         runner.intermediate_output_ttl = 0
 605         runner.secret_store = cwltool.secrets.SecretStore()
 606
 607         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 608         runner.api.collections().get().execute.return_value = {
 609             "portable_data_hash": "99999999999999999999999999999994+99",
 610             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 611
 612         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 613
 614         tool = cmap({
 615             "inputs": [
 616                 {"id": "p1",
 617                  "type": "Directory"}
 618             ],
 619             "outputs": [],
 620             "baseCommand": "ls",
 621             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 622             "id": "#",
 623             "class": "CommandLineTool"
 624         })
 625
 626         loadingContext, runtimeContext = self.helper(runner)
 627         runtimeContext.name = "test_run_mounts"
 628
 629         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 630         arvtool.formatgraph = None
 631         job_order = {
 632             "p1": {
 633                 "class": "Directory",
 634                 "location": "keep:99999999999999999999999999999994+44",
 635                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 636                 "listing": [
 637                     {
 638                         "class": "File",
 639                         "location": "keep:99999999999999999999999999999994+44/file1",
 640                     },
 641                     {
 642                         "class": "File",
 643                         "location": "keep:99999999999999999999999999999994+44/file2",
 644                     }
 645                 ]
 646             }
 647         }
 648         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 649             j.run(runtimeContext)
 650             runner.api.container_requests().create.assert_called_with(
 651                 body=JsonDiffMatcher({
 652                     'environment': {
 653                         'HOME': '/var/spool/cwl',
 654                         'TMPDIR': '/tmp'
 655                     },
 656                     'name': 'test_run_mounts',
 657                     'runtime_constraints': {
 658                         'vcpus': 1,
 659                         'ram': 1073741824
 660                     },
 661                     'use_existing': True,
 662                     'priority': 500,
 663                     'mounts': {
 664                         "/keep/99999999999999999999999999999994+44": {
 665                             "kind": "collection",
 666                             "portable_data_hash": "99999999999999999999999999999994+44",
 667                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 668                         },
 669                         '/tmp': {'kind': 'tmp',
 670                                  "capacity": 1073741824 },
 671                         '/var/spool/cwl': {'kind': 'tmp',
 672                                            "capacity": 1073741824 }
 673                     },
 674                     'state': 'Committed',
 675                     'output_name': 'Output for step test_run_mounts',
 676                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 677                     'output_path': '/var/spool/cwl',
 678                     'output_ttl': 0,
 679                     'container_image': '99999999999999999999999999999994+99',
 680                     'command': ['ls', '/var/spool/cwl'],
 681                     'cwd': '/var/spool/cwl',
 682                     'scheduling_parameters': {},
 683                     'properties': {},
 684                     'secret_mounts': {}
 685                 }))
 686
 687     # The test passes no builder.resources
 688     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 689     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 690     def test_secrets(self, keepdocker):
 691         arv_docker_clear_cache()
 692
 693         runner = mock.MagicMock()
 694         runner.ignore_docker_for_reuse = False
 695         runner.intermediate_output_ttl = 0
 696         runner.secret_store = cwltool.secrets.SecretStore()
 697
 698         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 699         runner.api.collections().get().execute.return_value = {
 700             "portable_data_hash": "99999999999999999999999999999993+99"}
 701
 702         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 703
 704         tool = cmap({"arguments": ["md5sum", "example.conf"],
 705                      "class": "CommandLineTool",
 706                      "hints": [
 707                          {
 708                              "class": "http://commonwl.org/cwltool#Secrets",
 709                              "secrets": [
 710                                  "#secret_job.cwl/pw"
 711                              ]
 712                          }
 713                      ],
 714                      "id": "#secret_job.cwl",
 715                      "inputs": [
 716                          {
 717                              "id": "#secret_job.cwl/pw",
 718                              "type": "string"
 719                          }
 720                      ],
 721                      "outputs": [
 722                      ],
 723                      "requirements": [
 724                          {
 725                              "class": "InitialWorkDirRequirement",
 726                              "listing": [
 727                                  {
 728                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 729                                      "entryname": "example.conf"
 730                                  }
 731                              ]
 732                          }
 733                      ]})
 734
 735         loadingContext, runtimeContext = self.helper(runner)
 736         runtimeContext.name = "test_secrets"
 737
 738         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 739         arvtool.formatgraph = None
 740
 741         job_order = {"pw": "blorp"}
 742         runner.secret_store.store(["pw"], job_order)
 743
 744         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 745             j.run(runtimeContext)
 746             runner.api.container_requests().create.assert_called_with(
 747                 body=JsonDiffMatcher({
 748                     'environment': {
 749                         'HOME': '/var/spool/cwl',
 750                         'TMPDIR': '/tmp'
 751                     },
 752                     'name': 'test_secrets',
 753                     'runtime_constraints': {
 754                         'vcpus': 1,
 755                         'ram': 1073741824
 756                     },
 757                     'use_existing': True,
 758                     'priority': 500,
 759                     'mounts': {
 760                         '/tmp': {'kind': 'tmp',
 761                                  "capacity": 1073741824
 762                              },
 763                         '/var/spool/cwl': {'kind': 'tmp',
 764                                            "capacity": 1073741824 }
 765                     },
 766                     'state': 'Committed',
 767                     'output_name': 'Output for step test_secrets',
 768                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 769                     'output_path': '/var/spool/cwl',
 770                     'output_ttl': 0,
 771                     'container_image': '99999999999999999999999999999993+99',
 772                     'command': ['md5sum', 'example.conf'],
 773                     'cwd': '/var/spool/cwl',
 774                     'scheduling_parameters': {},
 775                     'properties': {},
 776                     "secret_mounts": {
 777                         "/var/spool/cwl/example.conf": {
 778                             "content": "username: user\npassword: blorp\n",
 779                             "kind": "text"
 780                         }
 781                     }
 782                 }))
 783
 784     # The test passes no builder.resources
 785     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 786     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 787     def test_timelimit(self, keepdocker):
 788         arv_docker_clear_cache()
 789
 790         runner = mock.MagicMock()
 791         runner.ignore_docker_for_reuse = False
 792         runner.intermediate_output_ttl = 0
 793         runner.secret_store = cwltool.secrets.SecretStore()
 794
 795         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 796         runner.api.collections().get().execute.return_value = {
 797             "portable_data_hash": "99999999999999999999999999999993+99"}
 798
 799         tool = cmap({
 800             "inputs": [],
 801             "outputs": [],
 802             "baseCommand": "ls",
 803             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 804             "id": "#",
 805             "class": "CommandLineTool",
 806             "hints": [
 807                 {
 808                     "class": "ToolTimeLimit",
 809                     "timelimit": 42
 810                 }
 811             ]
 812         })
 813
 814         loadingContext, runtimeContext = self.helper(runner)
 815         runtimeContext.name = "test_timelimit"
 816
 817         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 818         arvtool.formatgraph = None
 819
 820         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 821             j.run(runtimeContext)
 822
 823         _, kwargs = runner.api.container_requests().create.call_args
 824         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 825
 826
 827 class TestWorkflow(unittest.TestCase):
 828     def setUp(self):
 829         cwltool.process._names = set()
 830
 831     def helper(self, runner, enable_reuse=True):
 832         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 833
 834         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 835                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 836
 837         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 838         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 839         document_loader.fetch_text = document_loader.fetcher.fetch_text
 840         document_loader.check_exists = document_loader.fetcher.check_exists
 841
 842         loadingContext = arvados_cwl.context.ArvLoadingContext(
 843             {"avsc_names": avsc_names,
 844              "basedir": "",
 845              "make_fs_access": make_fs_access,
 846              "loader": document_loader,
 847              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 848              "construct_tool_object": runner.arv_make_tool})
 849         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 850             {"work_api": "containers",
 851              "basedir": "",
 852              "name": "test_run_wf_"+str(enable_reuse),
 853              "make_fs_access": make_fs_access,
 854              "tmpdir": "/tmp",
 855              "enable_reuse": enable_reuse,
 856              "priority": 500})
 857
 858         return loadingContext, runtimeContext
 859
 860     # The test passes no builder.resources
 861     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 862     @mock.patch("arvados.collection.CollectionReader")
 863     @mock.patch("arvados.collection.Collection")
 864     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 865     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 866         arv_docker_clear_cache()
 867         arvados_cwl.add_arv_hints()
 868
 869         api = mock.MagicMock()
 870         api._rootDesc = get_rootDesc()
 871
 872         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 873         self.assertEqual(runner.work_api, 'containers')
 874
 875         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 876         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 877         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 878                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 879
 880         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 881         runner.ignore_docker_for_reuse = False
 882         runner.num_retries = 0
 883         runner.secret_store = cwltool.secrets.SecretStore()
 884
 885         loadingContext, runtimeContext = self.helper(runner)
 886         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 887
 888         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 889         metadata["cwlVersion"] = tool["cwlVersion"]
 890
 891         mockc = mock.MagicMock()
 892         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 893         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 894
 895         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 896         arvtool.formatgraph = None
 897         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 898
 899         next(it).run(runtimeContext)
 900         next(it).run(runtimeContext)
 901
 902         with open("tests/wf/scatter2_subwf.cwl") as f:
 903             subwf = StripYAMLComments(f.read()).rstrip()
 904
 905         runner.api.container_requests().create.assert_called_with(
 906             body=JsonDiffMatcher({
 907                 "command": [
 908                     "cwltool",
 909                     "--no-container",
 910                     "--move-outputs",
 911                     "--preserve-entire-environment",
 912                     "workflow.cwl#main",
 913                     "cwl.input.yml"
 914                 ],
 915                 "container_image": "99999999999999999999999999999993+99",
 916                 "cwd": "/var/spool/cwl",
 917                 "environment": {
 918                     "HOME": "/var/spool/cwl",
 919                     "TMPDIR": "/tmp"
 920                 },
 921                 "mounts": {
 922                     "/keep/99999999999999999999999999999999+118": {
 923                         "kind": "collection",
 924                         "portable_data_hash": "99999999999999999999999999999999+118"
 925                     },
 926                     "/tmp": {
 927                         "capacity": 1073741824,
 928                         "kind": "tmp"
 929                     },
 930                     "/var/spool/cwl": {
 931                         "capacity": 1073741824,
 932                         "kind": "tmp"
 933                     },
 934                     "/var/spool/cwl/cwl.input.yml": {
 935                         "kind": "collection",
 936                         "path": "cwl.input.yml",
 937                         "portable_data_hash": "99999999999999999999999999999996+99"
 938                     },
 939                     "/var/spool/cwl/workflow.cwl": {
 940                         "kind": "collection",
 941                         "path": "workflow.cwl",
 942                         "portable_data_hash": "99999999999999999999999999999996+99"
 943                     },
 944                     "stdout": {
 945                         "kind": "file",
 946                         "path": "/var/spool/cwl/cwl.output.json"
 947                     }
 948                 },
 949                 "name": "scatterstep",
 950                 "output_name": "Output for step scatterstep",
 951                 "output_path": "/var/spool/cwl",
 952                 "output_ttl": 0,
 953                 "priority": 500,
 954                 "properties": {},
 955                 "runtime_constraints": {
 956                     "ram": 1073741824,
 957                     "vcpus": 1
 958                 },
 959                 "scheduling_parameters": {},
 960                 "secret_mounts": {},
 961                 "state": "Committed",
 962                 "use_existing": True
 963             }))
 964         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 965         mockc.open().__enter__().write.assert_has_calls([mock.call(
 966 '''{
 967   "fileblub": {
 968     "basename": "token.txt",
 969     "class": "File",
 970     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 971     "size": 0
 972   },
 973   "sleeptime": 5
 974 }''')])
 975
 976     # The test passes no builder.resources
 977     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 978     @mock.patch("arvados.collection.CollectionReader")
 979     @mock.patch("arvados.collection.Collection")
 980     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 981     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 982         arv_docker_clear_cache()
 983         arvados_cwl.add_arv_hints()
 984
 985         api = mock.MagicMock()
 986         api._rootDesc = get_rootDesc()
 987
 988         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 989         self.assertEqual(runner.work_api, 'containers')
 990
 991         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 992         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 993                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
 994         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 995                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 996
 997         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 998         runner.ignore_docker_for_reuse = False
 999         runner.num_retries = 0
1000         runner.secret_store = cwltool.secrets.SecretStore()
1001
1002         loadingContext, runtimeContext = self.helper(runner)
1003         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1004         loadingContext.do_update = True
1005         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1006
1007         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1008
1009         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1010         arvtool.formatgraph = None
1011         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1012
1013         next(it).run(runtimeContext)
1014         next(it).run(runtimeContext)
1015
1016         with open("tests/wf/echo-subwf.cwl") as f:
1017             subwf = StripYAMLComments(f.read())
1018
1019         runner.api.container_requests().create.assert_called_with(
1020             body=JsonDiffMatcher({
1021                 'output_ttl': 0,
1022                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1023                 'scheduling_parameters': {},
1024                 'name': u'echo-subwf',
1025                 'secret_mounts': {},
1026                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1027                 'properties': {},
1028                 'priority': 500,
1029                 'mounts': {
1030                     '/var/spool/cwl/cwl.input.yml': {
1031                         'portable_data_hash': '99999999999999999999999999999996+99',
1032                         'kind': 'collection',
1033                         'path': 'cwl.input.yml'
1034                     },
1035                     '/var/spool/cwl/workflow.cwl': {
1036                         'portable_data_hash': '99999999999999999999999999999996+99',
1037                         'kind': 'collection',
1038                         'path': 'workflow.cwl'
1039                     },
1040                     'stdout': {
1041                         'path': '/var/spool/cwl/cwl.output.json',
1042                         'kind': 'file'
1043                     },
1044                     '/tmp': {
1045                         'kind': 'tmp',
1046                         'capacity': 1073741824
1047                     }, '/var/spool/cwl': {
1048                         'kind': 'tmp',
1049                         'capacity': 3221225472
1050                     }
1051                 },
1052                 'state': 'Committed',
1053                 'output_path': '/var/spool/cwl',
1054                 'container_image': '99999999999999999999999999999993+99',
1055                 'command': [
1056                     u'cwltool',
1057                     u'--no-container',
1058                     u'--move-outputs',
1059                     u'--preserve-entire-environment',
1060                     u'workflow.cwl#main',
1061                     u'cwl.input.yml'
1062                 ],
1063                 'use_existing': True,
1064                 'output_name': u'Output for step echo-subwf',
1065                 'cwd': '/var/spool/cwl'
1066             }))
1067
1068     def test_default_work_api(self):
1069         arvados_cwl.add_arv_hints()
1070
1071         api = mock.MagicMock()
1072         api._rootDesc = copy.deepcopy(get_rootDesc())
1073         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1074         self.assertEqual(runner.work_api, 'containers')