sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher, StripYAMLComments
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def setUp(self):
  60         cwltool.process._names = set()
  61
  62     def helper(self, runner, enable_reuse=True):
  63         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
  64
  65         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  66                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  67         loadingContext = arvados_cwl.context.ArvLoadingContext(
  68             {"avsc_names": avsc_names,
  69              "basedir": "",
  70              "make_fs_access": make_fs_access,
  71              "loader": Loader({}),
  72              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  73         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  74             {"work_api": "containers",
  75              "basedir": "",
  76              "name": "test_run_"+str(enable_reuse),
  77              "make_fs_access": make_fs_access,
  78              "tmpdir": "/tmp",
  79              "enable_reuse": enable_reuse,
  80              "priority": 500,
  81              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82             })
  83
  84         return loadingContext, runtimeContext
  85
  86     # Helper function to set up the ArvCwlExecutor to use the containers api
  87     # and test that the RuntimeStatusLoggingHandler is set up correctly
  88     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  89         api = mock.MagicMock()
  90         api._rootDesc = copy.deepcopy(get_rootDesc())
  91
  92         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  93         # adds the logging handler that will call runtime_status_update() mock
  94         self.assertFalse(gcc_mock.called)
  95         runner = arvados_cwl.ArvCwlExecutor(api)
  96         self.assertEqual(runner.work_api, 'containers')
  97         root_logger = logging.getLogger('')
  98         handlerClasses = [h.__class__ for h in root_logger.handlers]
  99         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 100         return runner
 101
 102     # The test passes no builder.resources
 103     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 104     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 105     def test_run(self, keepdocker):
 106         for enable_reuse in (True, False):
 107             arv_docker_clear_cache()
 108
 109             runner = mock.MagicMock()
 110             runner.ignore_docker_for_reuse = False
 111             runner.intermediate_output_ttl = 0
 112             runner.secret_store = cwltool.secrets.SecretStore()
 113
 114             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 115             runner.api.collections().get().execute.return_value = {
 116                 "portable_data_hash": "99999999999999999999999999999993+99"}
 117
 118             tool = cmap({
 119                 "inputs": [],
 120                 "outputs": [],
 121                 "baseCommand": "ls",
 122                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 123                 "id": "#",
 124                 "class": "CommandLineTool"
 125             })
 126
 127             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 128
 129             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 130             arvtool.formatgraph = None
 131
 132             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 133                 j.run(runtimeContext)
 134                 runner.api.container_requests().create.assert_called_with(
 135                     body=JsonDiffMatcher({
 136                         'environment': {
 137                             'HOME': '/var/spool/cwl',
 138                             'TMPDIR': '/tmp'
 139                         },
 140                         'name': 'test_run_'+str(enable_reuse),
 141                         'runtime_constraints': {
 142                             'vcpus': 1,
 143                             'ram': 1073741824
 144                         },
 145                         'use_existing': enable_reuse,
 146                         'priority': 500,
 147                         'mounts': {
 148                             '/tmp': {'kind': 'tmp',
 149                                      "capacity": 1073741824
 150                                  },
 151                             '/var/spool/cwl': {'kind': 'tmp',
 152                                                "capacity": 1073741824 }
 153                         },
 154                         'state': 'Committed',
 155                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 156                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 157                         'output_path': '/var/spool/cwl',
 158                         'output_ttl': 0,
 159                         'container_image': '99999999999999999999999999999993+99',
 160                         'command': ['ls', '/var/spool/cwl'],
 161                         'cwd': '/var/spool/cwl',
 162                         'scheduling_parameters': {},
 163                         'properties': {},
 164                         'secret_mounts': {}
 165                     }))
 166
 167     # The test passes some fields in builder.resources
 168     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 169     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 170     def test_resource_requirements(self, keepdocker):
 171         arv_docker_clear_cache()
 172         runner = mock.MagicMock()
 173         runner.ignore_docker_for_reuse = False
 174         runner.intermediate_output_ttl = 3600
 175         runner.secret_store = cwltool.secrets.SecretStore()
 176
 177         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 178         runner.api.collections().get().execute.return_value = {
 179             "portable_data_hash": "99999999999999999999999999999993+99"}
 180
 181         tool = cmap({
 182             "inputs": [],
 183             "outputs": [],
 184             "hints": [{
 185                 "class": "ResourceRequirement",
 186                 "coresMin": 3,
 187                 "ramMin": 3000,
 188                 "tmpdirMin": 4000,
 189                 "outdirMin": 5000
 190             }, {
 191                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 192                 "keep_cache": 512
 193             }, {
 194                 "class": "http://arvados.org/cwl#APIRequirement",
 195             }, {
 196                 "class": "http://arvados.org/cwl#PartitionRequirement",
 197                 "partition": "blurb"
 198             }, {
 199                 "class": "http://arvados.org/cwl#IntermediateOutput",
 200                 "outputTTL": 7200
 201             }, {
 202                 "class": "http://arvados.org/cwl#ReuseRequirement",
 203                 "enableReuse": False
 204             }],
 205             "baseCommand": "ls",
 206             "id": "#",
 207             "class": "CommandLineTool"
 208         })
 209
 210         loadingContext, runtimeContext = self.helper(runner)
 211         runtimeContext.name = "test_resource_requirements"
 212
 213         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 214         arvtool.formatgraph = None
 215         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 216             j.run(runtimeContext)
 217
 218         call_args, call_kwargs = runner.api.container_requests().create.call_args
 219
 220         call_body_expected = {
 221             'environment': {
 222                 'HOME': '/var/spool/cwl',
 223                 'TMPDIR': '/tmp'
 224             },
 225             'name': 'test_resource_requirements',
 226             'runtime_constraints': {
 227                 'vcpus': 3,
 228                 'ram': 3145728000,
 229                 'keep_cache_ram': 536870912,
 230                 'API': True
 231             },
 232             'use_existing': False,
 233             'priority': 500,
 234             'mounts': {
 235                 '/tmp': {'kind': 'tmp',
 236                          "capacity": 4194304000 },
 237                 '/var/spool/cwl': {'kind': 'tmp',
 238                                    "capacity": 5242880000 }
 239             },
 240             'state': 'Committed',
 241             'output_name': 'Output for step test_resource_requirements',
 242             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 243             'output_path': '/var/spool/cwl',
 244             'output_ttl': 7200,
 245             'container_image': '99999999999999999999999999999993+99',
 246             'command': ['ls'],
 247             'cwd': '/var/spool/cwl',
 248             'scheduling_parameters': {
 249                 'partitions': ['blurb']
 250             },
 251             'properties': {},
 252             'secret_mounts': {}
 253         }
 254
 255         call_body = call_kwargs.get('body', None)
 256         self.assertNotEqual(None, call_body)
 257         for key in call_body:
 258             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 259
 260
 261     # The test passes some fields in builder.resources
 262     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 263     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 264     @mock.patch("arvados.collection.Collection")
 265     def test_initial_work_dir(self, collection_mock, keepdocker):
 266         arv_docker_clear_cache()
 267         runner = mock.MagicMock()
 268         runner.ignore_docker_for_reuse = False
 269         runner.intermediate_output_ttl = 0
 270         runner.secret_store = cwltool.secrets.SecretStore()
 271
 272         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 273         runner.api.collections().get().execute.return_value = {
 274             "portable_data_hash": "99999999999999999999999999999993+99"}
 275
 276         sourcemock = mock.MagicMock()
 277         def get_collection_mock(p):
 278             if "/" in p:
 279                 return (sourcemock, p.split("/", 1)[1])
 280             else:
 281                 return (sourcemock, "")
 282         runner.fs_access.get_collection.side_effect = get_collection_mock
 283
 284         vwdmock = mock.MagicMock()
 285         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 286
 287         tool = cmap({
 288             "inputs": [],
 289             "outputs": [],
 290             "hints": [{
 291                 "class": "InitialWorkDirRequirement",
 292                 "listing": [{
 293                     "class": "File",
 294                     "basename": "foo",
 295                     "location": "keep:99999999999999999999999999999995+99/bar"
 296                 },
 297                 {
 298                     "class": "Directory",
 299                     "basename": "foo2",
 300                     "location": "keep:99999999999999999999999999999995+99"
 301                 },
 302                 {
 303                     "class": "File",
 304                     "basename": "filename",
 305                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 306                 },
 307                 {
 308                     "class": "Directory",
 309                     "basename": "subdir",
 310                     "location": "keep:99999999999999999999999999999995+99/subdir"
 311                 }                        ]
 312             }],
 313             "baseCommand": "ls",
 314             "id": "#",
 315             "class": "CommandLineTool"
 316         })
 317
 318         loadingContext, runtimeContext = self.helper(runner)
 319         runtimeContext.name = "test_initial_work_dir"
 320
 321         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 322         arvtool.formatgraph = None
 323         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 324             j.run(runtimeContext)
 325
 326         call_args, call_kwargs = runner.api.container_requests().create.call_args
 327
 328         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 329         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 330         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 332
 333         call_body_expected = {
 334             'environment': {
 335                 'HOME': '/var/spool/cwl',
 336                 'TMPDIR': '/tmp'
 337             },
 338             'name': 'test_initial_work_dir',
 339             'runtime_constraints': {
 340                 'vcpus': 1,
 341                 'ram': 1073741824
 342             },
 343             'use_existing': True,
 344             'priority': 500,
 345             'mounts': {
 346                 '/tmp': {'kind': 'tmp',
 347                          "capacity": 1073741824 },
 348                 '/var/spool/cwl': {'kind': 'tmp',
 349                                    "capacity": 1073741824 },
 350                 '/var/spool/cwl/foo': {
 351                     'kind': 'collection',
 352                     'path': 'foo',
 353                     'portable_data_hash': '99999999999999999999999999999996+99'
 354                 },
 355                 '/var/spool/cwl/foo2': {
 356                     'kind': 'collection',
 357                     'path': 'foo2',
 358                     'portable_data_hash': '99999999999999999999999999999996+99'
 359                 },
 360                 '/var/spool/cwl/filename': {
 361                     'kind': 'collection',
 362                     'path': 'filename',
 363                     'portable_data_hash': '99999999999999999999999999999996+99'
 364                 },
 365                 '/var/spool/cwl/subdir': {
 366                     'kind': 'collection',
 367                     'path': 'subdir',
 368                     'portable_data_hash': '99999999999999999999999999999996+99'
 369                 }
 370             },
 371             'state': 'Committed',
 372             'output_name': 'Output for step test_initial_work_dir',
 373             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 374             'output_path': '/var/spool/cwl',
 375             'output_ttl': 0,
 376             'container_image': '99999999999999999999999999999993+99',
 377             'command': ['ls'],
 378             'cwd': '/var/spool/cwl',
 379             'scheduling_parameters': {
 380             },
 381             'properties': {},
 382             'secret_mounts': {}
 383         }
 384
 385         call_body = call_kwargs.get('body', None)
 386         self.assertNotEqual(None, call_body)
 387         for key in call_body:
 388             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 389
 390
 391     # Test redirecting stdin/stdout/stderr
 392     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 393     def test_redirects(self, keepdocker):
 394         arv_docker_clear_cache()
 395
 396         runner = mock.MagicMock()
 397         runner.ignore_docker_for_reuse = False
 398         runner.intermediate_output_ttl = 0
 399         runner.secret_store = cwltool.secrets.SecretStore()
 400
 401         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 402         runner.api.collections().get().execute.return_value = {
 403             "portable_data_hash": "99999999999999999999999999999993+99"}
 404
 405         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 406
 407         tool = cmap({
 408             "inputs": [],
 409             "outputs": [],
 410             "baseCommand": "ls",
 411             "stdout": "stdout.txt",
 412             "stderr": "stderr.txt",
 413             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 414             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 415             "id": "#",
 416             "class": "CommandLineTool"
 417         })
 418
 419         loadingContext, runtimeContext = self.helper(runner)
 420         runtimeContext.name = "test_run_redirect"
 421
 422         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 423         arvtool.formatgraph = None
 424         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 425             j.run(runtimeContext)
 426             runner.api.container_requests().create.assert_called_with(
 427                 body=JsonDiffMatcher({
 428                     'environment': {
 429                         'HOME': '/var/spool/cwl',
 430                         'TMPDIR': '/tmp'
 431                     },
 432                     'name': 'test_run_redirect',
 433                     'runtime_constraints': {
 434                         'vcpus': 1,
 435                         'ram': 1073741824
 436                     },
 437                     'use_existing': True,
 438                     'priority': 500,
 439                     'mounts': {
 440                         '/tmp': {'kind': 'tmp',
 441                                  "capacity": 1073741824 },
 442                         '/var/spool/cwl': {'kind': 'tmp',
 443                                            "capacity": 1073741824 },
 444                         "stderr": {
 445                             "kind": "file",
 446                             "path": "/var/spool/cwl/stderr.txt"
 447                         },
 448                         "stdin": {
 449                             "kind": "collection",
 450                             "path": "file.txt",
 451                             "portable_data_hash": "99999999999999999999999999999996+99"
 452                         },
 453                         "stdout": {
 454                             "kind": "file",
 455                             "path": "/var/spool/cwl/stdout.txt"
 456                         },
 457                     },
 458                     'state': 'Committed',
 459                     "output_name": "Output for step test_run_redirect",
 460                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 461                     'output_path': '/var/spool/cwl',
 462                     'output_ttl': 0,
 463                     'container_image': '99999999999999999999999999999993+99',
 464                     'command': ['ls', '/var/spool/cwl'],
 465                     'cwd': '/var/spool/cwl',
 466                     'scheduling_parameters': {},
 467                     'properties': {},
 468                     'secret_mounts': {}
 469                 }))
 470
 471     @mock.patch("arvados.collection.Collection")
 472     def test_done(self, col):
 473         api = mock.MagicMock()
 474
 475         runner = mock.MagicMock()
 476         runner.api = api
 477         runner.num_retries = 0
 478         runner.ignore_docker_for_reuse = False
 479         runner.intermediate_output_ttl = 0
 480         runner.secret_store = cwltool.secrets.SecretStore()
 481
 482         runner.api.containers().get().execute.return_value = {"state":"Complete",
 483                                                               "output": "abc+123",
 484                                                               "exit_code": 0}
 485
 486         col().open.return_value = []
 487
 488         loadingContext, runtimeContext = self.helper(runner)
 489
 490         arvjob = arvados_cwl.ArvadosContainer(runner,
 491                                               runtimeContext,
 492                                               mock.MagicMock(),
 493                                               {},
 494                                               None,
 495                                               [],
 496                                               [],
 497                                               "testjob")
 498         arvjob.output_callback = mock.MagicMock()
 499         arvjob.collect_outputs = mock.MagicMock()
 500         arvjob.successCodes = [0]
 501         arvjob.outdir = "/var/spool/cwl"
 502         arvjob.output_ttl = 3600
 503
 504         arvjob.collect_outputs.return_value = {"out": "stuff"}
 505
 506         arvjob.done({
 507             "state": "Final",
 508             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 509             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 510             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 511             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 512             "modified_at": "2017-05-26T12:01:22Z"
 513         })
 514
 515         self.assertFalse(api.collections().create.called)
 516         self.assertFalse(runner.runtime_status_error.called)
 517
 518         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 519         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 520         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 521
 522     # Test to make sure we dont call runtime_status_update if we already did
 523     # some where higher up in the call stack
 524     @mock.patch("arvados_cwl.util.get_current_container")
 525     def test_recursive_runtime_status_update(self, gcc_mock):
 526         self.setup_and_test_container_executor_and_logging(gcc_mock)
 527         root_logger = logging.getLogger('')
 528
 529         # get_current_container is invoked when we call runtime_status_update
 530         # so try and log again!
 531         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 532         try:
 533             root_logger.error("First Error")
 534         except RuntimeError:
 535             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 536
 537
 538     # Test to make sure that an exception raised from
 539     # get_current_container doesn't cause the logger to raise an
 540     # exception
 541     @mock.patch("arvados_cwl.util.get_current_container")
 542     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 543         self.setup_and_test_container_executor_and_logging(gcc_mock)
 544         root_logger = logging.getLogger('')
 545
 546         # get_current_container is invoked when we call
 547         # runtime_status_update, it is going to also raise an
 548         # exception.
 549         gcc_mock.side_effect = Exception("Second Error")
 550         try:
 551             root_logger.error("First Error")
 552         except Exception:
 553             self.fail("Exception in logger should not propagate")
 554         self.assertTrue(gcc_mock.called)
 555
 556     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 557     @mock.patch("arvados_cwl.util.get_current_container")
 558     @mock.patch("arvados.collection.CollectionReader")
 559     @mock.patch("arvados.collection.Collection")
 560     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 561         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 562
 563         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 564         self.assertTrue(gcc_mock.called)
 565
 566         runner.num_retries = 0
 567         runner.ignore_docker_for_reuse = False
 568         runner.intermediate_output_ttl = 0
 569         runner.secret_store = cwltool.secrets.SecretStore()
 570         runner.label = mock.MagicMock()
 571         runner.label.return_value = '[container testjob]'
 572
 573         runner.api.containers().get().execute.return_value = {
 574             "state":"Complete",
 575             "output": "abc+123",
 576             "exit_code": 1,
 577             "log": "def+234"
 578         }
 579
 580         col().open.return_value = []
 581
 582         loadingContext, runtimeContext = self.helper(runner)
 583
 584         arvjob = arvados_cwl.ArvadosContainer(runner,
 585                                               runtimeContext,
 586                                               mock.MagicMock(),
 587                                               {},
 588                                               None,
 589                                               [],
 590                                               [],
 591                                               "testjob")
 592         arvjob.output_callback = mock.MagicMock()
 593         arvjob.collect_outputs = mock.MagicMock()
 594         arvjob.successCodes = [0]
 595         arvjob.outdir = "/var/spool/cwl"
 596         arvjob.output_ttl = 3600
 597         arvjob.collect_outputs.return_value = {"out": "stuff"}
 598
 599         arvjob.done({
 600             "state": "Final",
 601             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 602             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 603             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 604             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 605             "modified_at": "2017-05-26T12:01:22Z"
 606         })
 607
 608         rts_mock.assert_called_with(
 609             'error',
 610             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 611             '  ** log is empty **'
 612         )
 613         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 614
 615     # The test passes no builder.resources
 616     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 617     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 618     def test_mounts(self, keepdocker):
 619         arv_docker_clear_cache()
 620
 621         runner = mock.MagicMock()
 622         runner.ignore_docker_for_reuse = False
 623         runner.intermediate_output_ttl = 0
 624         runner.secret_store = cwltool.secrets.SecretStore()
 625
 626         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 627         runner.api.collections().get().execute.return_value = {
 628             "portable_data_hash": "99999999999999999999999999999994+99",
 629             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 630
 631         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 632
 633         tool = cmap({
 634             "inputs": [
 635                 {"id": "p1",
 636                  "type": "Directory"}
 637             ],
 638             "outputs": [],
 639             "baseCommand": "ls",
 640             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 641             "id": "#",
 642             "class": "CommandLineTool"
 643         })
 644
 645         loadingContext, runtimeContext = self.helper(runner)
 646         runtimeContext.name = "test_run_mounts"
 647
 648         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 649         arvtool.formatgraph = None
 650         job_order = {
 651             "p1": {
 652                 "class": "Directory",
 653                 "location": "keep:99999999999999999999999999999994+44",
 654                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 655                 "listing": [
 656                     {
 657                         "class": "File",
 658                         "location": "keep:99999999999999999999999999999994+44/file1",
 659                     },
 660                     {
 661                         "class": "File",
 662                         "location": "keep:99999999999999999999999999999994+44/file2",
 663                     }
 664                 ]
 665             }
 666         }
 667         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 668             j.run(runtimeContext)
 669             runner.api.container_requests().create.assert_called_with(
 670                 body=JsonDiffMatcher({
 671                     'environment': {
 672                         'HOME': '/var/spool/cwl',
 673                         'TMPDIR': '/tmp'
 674                     },
 675                     'name': 'test_run_mounts',
 676                     'runtime_constraints': {
 677                         'vcpus': 1,
 678                         'ram': 1073741824
 679                     },
 680                     'use_existing': True,
 681                     'priority': 500,
 682                     'mounts': {
 683                         "/keep/99999999999999999999999999999994+44": {
 684                             "kind": "collection",
 685                             "portable_data_hash": "99999999999999999999999999999994+44",
 686                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 687                         },
 688                         '/tmp': {'kind': 'tmp',
 689                                  "capacity": 1073741824 },
 690                         '/var/spool/cwl': {'kind': 'tmp',
 691                                            "capacity": 1073741824 }
 692                     },
 693                     'state': 'Committed',
 694                     'output_name': 'Output for step test_run_mounts',
 695                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 696                     'output_path': '/var/spool/cwl',
 697                     'output_ttl': 0,
 698                     'container_image': '99999999999999999999999999999994+99',
 699                     'command': ['ls', '/var/spool/cwl'],
 700                     'cwd': '/var/spool/cwl',
 701                     'scheduling_parameters': {},
 702                     'properties': {},
 703                     'secret_mounts': {}
 704                 }))
 705
 706     # The test passes no builder.resources
 707     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 708     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 709     def test_secrets(self, keepdocker):
 710         arv_docker_clear_cache()
 711
 712         runner = mock.MagicMock()
 713         runner.ignore_docker_for_reuse = False
 714         runner.intermediate_output_ttl = 0
 715         runner.secret_store = cwltool.secrets.SecretStore()
 716
 717         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 718         runner.api.collections().get().execute.return_value = {
 719             "portable_data_hash": "99999999999999999999999999999993+99"}
 720
 721         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 722
 723         tool = cmap({"arguments": ["md5sum", "example.conf"],
 724                      "class": "CommandLineTool",
 725                      "hints": [
 726                          {
 727                              "class": "http://commonwl.org/cwltool#Secrets",
 728                              "secrets": [
 729                                  "#secret_job.cwl/pw"
 730                              ]
 731                          }
 732                      ],
 733                      "id": "#secret_job.cwl",
 734                      "inputs": [
 735                          {
 736                              "id": "#secret_job.cwl/pw",
 737                              "type": "string"
 738                          }
 739                      ],
 740                      "outputs": [
 741                      ],
 742                      "requirements": [
 743                          {
 744                              "class": "InitialWorkDirRequirement",
 745                              "listing": [
 746                                  {
 747                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 748                                      "entryname": "example.conf"
 749                                  }
 750                              ]
 751                          }
 752                      ]})
 753
 754         loadingContext, runtimeContext = self.helper(runner)
 755         runtimeContext.name = "test_secrets"
 756
 757         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 758         arvtool.formatgraph = None
 759
 760         job_order = {"pw": "blorp"}
 761         runner.secret_store.store(["pw"], job_order)
 762
 763         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 764             j.run(runtimeContext)
 765             runner.api.container_requests().create.assert_called_with(
 766                 body=JsonDiffMatcher({
 767                     'environment': {
 768                         'HOME': '/var/spool/cwl',
 769                         'TMPDIR': '/tmp'
 770                     },
 771                     'name': 'test_secrets',
 772                     'runtime_constraints': {
 773                         'vcpus': 1,
 774                         'ram': 1073741824
 775                     },
 776                     'use_existing': True,
 777                     'priority': 500,
 778                     'mounts': {
 779                         '/tmp': {'kind': 'tmp',
 780                                  "capacity": 1073741824
 781                              },
 782                         '/var/spool/cwl': {'kind': 'tmp',
 783                                            "capacity": 1073741824 }
 784                     },
 785                     'state': 'Committed',
 786                     'output_name': 'Output for step test_secrets',
 787                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 788                     'output_path': '/var/spool/cwl',
 789                     'output_ttl': 0,
 790                     'container_image': '99999999999999999999999999999993+99',
 791                     'command': ['md5sum', 'example.conf'],
 792                     'cwd': '/var/spool/cwl',
 793                     'scheduling_parameters': {},
 794                     'properties': {},
 795                     "secret_mounts": {
 796                         "/var/spool/cwl/example.conf": {
 797                             "content": "username: user\npassword: blorp\n",
 798                             "kind": "text"
 799                         }
 800                     }
 801                 }))
 802
 803     # The test passes no builder.resources
 804     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 805     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 806     def test_timelimit(self, keepdocker):
 807         arv_docker_clear_cache()
 808
 809         runner = mock.MagicMock()
 810         runner.ignore_docker_for_reuse = False
 811         runner.intermediate_output_ttl = 0
 812         runner.secret_store = cwltool.secrets.SecretStore()
 813
 814         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 815         runner.api.collections().get().execute.return_value = {
 816             "portable_data_hash": "99999999999999999999999999999993+99"}
 817
 818         tool = cmap({
 819             "inputs": [],
 820             "outputs": [],
 821             "baseCommand": "ls",
 822             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 823             "id": "#",
 824             "class": "CommandLineTool",
 825             "hints": [
 826                 {
 827                     "class": "ToolTimeLimit",
 828                     "timelimit": 42
 829                 }
 830             ]
 831         })
 832
 833         loadingContext, runtimeContext = self.helper(runner)
 834         runtimeContext.name = "test_timelimit"
 835
 836         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 837         arvtool.formatgraph = None
 838
 839         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 840             j.run(runtimeContext)
 841
 842         _, kwargs = runner.api.container_requests().create.call_args
 843         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 844
 845
 846 class TestWorkflow(unittest.TestCase):
 847     def setUp(self):
 848         cwltool.process._names = set()
 849
 850     def helper(self, runner, enable_reuse=True):
 851         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 852
 853         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 854                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 855
 856         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 857         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 858         document_loader.fetch_text = document_loader.fetcher.fetch_text
 859         document_loader.check_exists = document_loader.fetcher.check_exists
 860
 861         loadingContext = arvados_cwl.context.ArvLoadingContext(
 862             {"avsc_names": avsc_names,
 863              "basedir": "",
 864              "make_fs_access": make_fs_access,
 865              "loader": document_loader,
 866              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 867              "construct_tool_object": runner.arv_make_tool})
 868         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 869             {"work_api": "containers",
 870              "basedir": "",
 871              "name": "test_run_wf_"+str(enable_reuse),
 872              "make_fs_access": make_fs_access,
 873              "tmpdir": "/tmp",
 874              "enable_reuse": enable_reuse,
 875              "priority": 500})
 876
 877         return loadingContext, runtimeContext
 878
 879     # The test passes no builder.resources
 880     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 881     @mock.patch("arvados.collection.CollectionReader")
 882     @mock.patch("arvados.collection.Collection")
 883     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 884     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 885         arv_docker_clear_cache()
 886         arvados_cwl.add_arv_hints()
 887
 888         api = mock.MagicMock()
 889         api._rootDesc = get_rootDesc()
 890
 891         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 892         self.assertEqual(runner.work_api, 'containers')
 893
 894         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 895         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 896         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 897                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 898
 899         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 900         runner.ignore_docker_for_reuse = False
 901         runner.num_retries = 0
 902         runner.secret_store = cwltool.secrets.SecretStore()
 903
 904         loadingContext, runtimeContext = self.helper(runner)
 905         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 906
 907         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 908         metadata["cwlVersion"] = tool["cwlVersion"]
 909
 910         mockc = mock.MagicMock()
 911         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 912         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 913
 914         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 915         arvtool.formatgraph = None
 916         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 917
 918         next(it).run(runtimeContext)
 919         next(it).run(runtimeContext)
 920
 921         with open("tests/wf/scatter2_subwf.cwl") as f:
 922             subwf = StripYAMLComments(f.read()).rstrip()
 923
 924         runner.api.container_requests().create.assert_called_with(
 925             body=JsonDiffMatcher({
 926                 "command": [
 927                     "cwltool",
 928                     "--no-container",
 929                     "--move-outputs",
 930                     "--preserve-entire-environment",
 931                     "workflow.cwl#main",
 932                     "cwl.input.yml"
 933                 ],
 934                 "container_image": "99999999999999999999999999999993+99",
 935                 "cwd": "/var/spool/cwl",
 936                 "environment": {
 937                     "HOME": "/var/spool/cwl",
 938                     "TMPDIR": "/tmp"
 939                 },
 940                 "mounts": {
 941                     "/keep/99999999999999999999999999999999+118": {
 942                         "kind": "collection",
 943                         "portable_data_hash": "99999999999999999999999999999999+118"
 944                     },
 945                     "/tmp": {
 946                         "capacity": 1073741824,
 947                         "kind": "tmp"
 948                     },
 949                     "/var/spool/cwl": {
 950                         "capacity": 1073741824,
 951                         "kind": "tmp"
 952                     },
 953                     "/var/spool/cwl/cwl.input.yml": {
 954                         "kind": "collection",
 955                         "path": "cwl.input.yml",
 956                         "portable_data_hash": "99999999999999999999999999999996+99"
 957                     },
 958                     "/var/spool/cwl/workflow.cwl": {
 959                         "kind": "collection",
 960                         "path": "workflow.cwl",
 961                         "portable_data_hash": "99999999999999999999999999999996+99"
 962                     },
 963                     "stdout": {
 964                         "kind": "file",
 965                         "path": "/var/spool/cwl/cwl.output.json"
 966                     }
 967                 },
 968                 "name": "scatterstep",
 969                 "output_name": "Output for step scatterstep",
 970                 "output_path": "/var/spool/cwl",
 971                 "output_ttl": 0,
 972                 "priority": 500,
 973                 "properties": {},
 974                 "runtime_constraints": {
 975                     "ram": 1073741824,
 976                     "vcpus": 1
 977                 },
 978                 "scheduling_parameters": {},
 979                 "secret_mounts": {},
 980                 "state": "Committed",
 981                 "use_existing": True
 982             }))
 983         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 984         mockc.open().__enter__().write.assert_has_calls([mock.call(
 985 '''{
 986   "fileblub": {
 987     "basename": "token.txt",
 988     "class": "File",
 989     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 990     "size": 0
 991   },
 992   "sleeptime": 5
 993 }''')])
 994
 995     # The test passes no builder.resources
 996     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 997     @mock.patch("arvados.collection.CollectionReader")
 998     @mock.patch("arvados.collection.Collection")
 999     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1000     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1001         arv_docker_clear_cache()
1002         arvados_cwl.add_arv_hints()
1003
1004         api = mock.MagicMock()
1005         api._rootDesc = get_rootDesc()
1006
1007         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1008         self.assertEqual(runner.work_api, 'containers')
1009
1010         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1011         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1012                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1013         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1014                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1015
1016         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1017         runner.ignore_docker_for_reuse = False
1018         runner.num_retries = 0
1019         runner.secret_store = cwltool.secrets.SecretStore()
1020
1021         loadingContext, runtimeContext = self.helper(runner)
1022         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1023         loadingContext.do_update = True
1024         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1025
1026         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1027
1028         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1029         arvtool.formatgraph = None
1030         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1031
1032         next(it).run(runtimeContext)
1033         next(it).run(runtimeContext)
1034
1035         with open("tests/wf/echo-subwf.cwl") as f:
1036             subwf = StripYAMLComments(f.read())
1037
1038         runner.api.container_requests().create.assert_called_with(
1039             body=JsonDiffMatcher({
1040                 'output_ttl': 0,
1041                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1042                 'scheduling_parameters': {},
1043                 'name': u'echo-subwf',
1044                 'secret_mounts': {},
1045                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1046                 'properties': {},
1047                 'priority': 500,
1048                 'mounts': {
1049                     '/var/spool/cwl/cwl.input.yml': {
1050                         'portable_data_hash': '99999999999999999999999999999996+99',
1051                         'kind': 'collection',
1052                         'path': 'cwl.input.yml'
1053                     },
1054                     '/var/spool/cwl/workflow.cwl': {
1055                         'portable_data_hash': '99999999999999999999999999999996+99',
1056                         'kind': 'collection',
1057                         'path': 'workflow.cwl'
1058                     },
1059                     'stdout': {
1060                         'path': '/var/spool/cwl/cwl.output.json',
1061                         'kind': 'file'
1062                     },
1063                     '/tmp': {
1064                         'kind': 'tmp',
1065                         'capacity': 1073741824
1066                     }, '/var/spool/cwl': {
1067                         'kind': 'tmp',
1068                         'capacity': 3221225472
1069                     }
1070                 },
1071                 'state': 'Committed',
1072                 'output_path': '/var/spool/cwl',
1073                 'container_image': '99999999999999999999999999999993+99',
1074                 'command': [
1075                     u'cwltool',
1076                     u'--no-container',
1077                     u'--move-outputs',
1078                     u'--preserve-entire-environment',
1079                     u'workflow.cwl#main',
1080                     u'cwl.input.yml'
1081                 ],
1082                 'use_existing': True,
1083                 'output_name': u'Output for step echo-subwf',
1084                 'cwd': '/var/spool/cwl'
1085             }))
1086
1087     def test_default_work_api(self):
1088         arvados_cwl.add_arv_hints()
1089
1090         api = mock.MagicMock()
1091         api._rootDesc = copy.deepcopy(get_rootDesc())
1092         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1093         self.assertEqual(runner.work_api, 'containers')