sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from cwltool.update import INTERNAL_VERSION
  22 from schema_salad.ref_resolver import Loader
  23 from schema_salad.sourceline import cmap
  24
  25 from .matcher import JsonDiffMatcher, StripYAMLComments
  26 from .mock_discovery import get_rootDesc
  27
  28 if not os.getenv('ARVADOS_DEBUG'):
  29     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  30     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  31
  32 class CollectionMock(object):
  33     def __init__(self, vwdmock, *args, **kwargs):
  34         self.vwdmock = vwdmock
  35         self.count = 0
  36
  37     def open(self, *args, **kwargs):
  38         self.count += 1
  39         return self.vwdmock.open(*args, **kwargs)
  40
  41     def copy(self, *args, **kwargs):
  42         self.count += 1
  43         self.vwdmock.copy(*args, **kwargs)
  44
  45     def save_new(self, *args, **kwargs):
  46         pass
  47
  48     def __len__(self):
  49         return self.count
  50
  51     def portable_data_hash(self):
  52         if self.count == 0:
  53             return arvados.config.EMPTY_BLOCK_LOCATOR
  54         else:
  55             return "99999999999999999999999999999996+99"
  56
  57
  58 class TestContainer(unittest.TestCase):
  59
  60     def setUp(self):
  61         cwltool.process._names = set()
  62
  63     def helper(self, runner, enable_reuse=True):
  64         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  65
  66         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  67                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  68         loadingContext = arvados_cwl.context.ArvLoadingContext(
  69             {"avsc_names": avsc_names,
  70              "basedir": "",
  71              "make_fs_access": make_fs_access,
  72              "loader": Loader({}),
  73              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  74         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  75             {"work_api": "containers",
  76              "basedir": "",
  77              "name": "test_run_"+str(enable_reuse),
  78              "make_fs_access": make_fs_access,
  79              "tmpdir": "/tmp",
  80              "enable_reuse": enable_reuse,
  81              "priority": 500,
  82              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83             })
  84
  85         return loadingContext, runtimeContext
  86
  87     # Helper function to set up the ArvCwlExecutor to use the containers api
  88     # and test that the RuntimeStatusLoggingHandler is set up correctly
  89     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  90         api = mock.MagicMock()
  91         api._rootDesc = copy.deepcopy(get_rootDesc())
  92
  93         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  94         # adds the logging handler that will call runtime_status_update() mock
  95         self.assertFalse(gcc_mock.called)
  96         runner = arvados_cwl.ArvCwlExecutor(api)
  97         self.assertEqual(runner.work_api, 'containers')
  98         root_logger = logging.getLogger('')
  99         handlerClasses = [h.__class__ for h in root_logger.handlers]
 100         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 101         return runner
 102
 103     # The test passes no builder.resources
 104     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 105     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 106     def test_run(self, keepdocker):
 107         for enable_reuse in (True, False):
 108             arv_docker_clear_cache()
 109
 110             runner = mock.MagicMock()
 111             runner.ignore_docker_for_reuse = False
 112             runner.intermediate_output_ttl = 0
 113             runner.secret_store = cwltool.secrets.SecretStore()
 114
 115             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 116             runner.api.collections().get().execute.return_value = {
 117                 "portable_data_hash": "99999999999999999999999999999993+99"}
 118
 119             tool = cmap({
 120                 "inputs": [],
 121                 "outputs": [],
 122                 "baseCommand": "ls",
 123                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 124                 "id": "#",
 125                 "class": "CommandLineTool"
 126             })
 127
 128             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 129
 130             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 131             arvtool.formatgraph = None
 132
 133             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 134                 j.run(runtimeContext)
 135                 runner.api.container_requests().create.assert_called_with(
 136                     body=JsonDiffMatcher({
 137                         'environment': {
 138                             'HOME': '/var/spool/cwl',
 139                             'TMPDIR': '/tmp'
 140                         },
 141                         'name': 'test_run_'+str(enable_reuse),
 142                         'runtime_constraints': {
 143                             'vcpus': 1,
 144                             'ram': 1073741824
 145                         },
 146                         'use_existing': enable_reuse,
 147                         'priority': 500,
 148                         'mounts': {
 149                             '/tmp': {'kind': 'tmp',
 150                                      "capacity": 1073741824
 151                                  },
 152                             '/var/spool/cwl': {'kind': 'tmp',
 153                                                "capacity": 1073741824 }
 154                         },
 155                         'state': 'Committed',
 156                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 157                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 158                         'output_path': '/var/spool/cwl',
 159                         'output_ttl': 0,
 160                         'container_image': '99999999999999999999999999999993+99',
 161                         'command': ['ls', '/var/spool/cwl'],
 162                         'cwd': '/var/spool/cwl',
 163                         'scheduling_parameters': {},
 164                         'properties': {},
 165                         'secret_mounts': {}
 166                     }))
 167
 168     # The test passes some fields in builder.resources
 169     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 170     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 171     def test_resource_requirements(self, keepdocker):
 172         arv_docker_clear_cache()
 173         runner = mock.MagicMock()
 174         runner.ignore_docker_for_reuse = False
 175         runner.intermediate_output_ttl = 3600
 176         runner.secret_store = cwltool.secrets.SecretStore()
 177
 178         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 179         runner.api.collections().get().execute.return_value = {
 180             "portable_data_hash": "99999999999999999999999999999993+99"}
 181
 182         tool = cmap({
 183             "inputs": [],
 184             "outputs": [],
 185             "hints": [{
 186                 "class": "ResourceRequirement",
 187                 "coresMin": 3,
 188                 "ramMin": 3000,
 189                 "tmpdirMin": 4000,
 190                 "outdirMin": 5000
 191             }, {
 192                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 193                 "keep_cache": 512
 194             }, {
 195                 "class": "http://arvados.org/cwl#APIRequirement",
 196             }, {
 197                 "class": "http://arvados.org/cwl#PartitionRequirement",
 198                 "partition": "blurb"
 199             }, {
 200                 "class": "http://arvados.org/cwl#IntermediateOutput",
 201                 "outputTTL": 7200
 202             }, {
 203                 "class": "http://arvados.org/cwl#ReuseRequirement",
 204                 "enableReuse": False
 205             }],
 206             "baseCommand": "ls",
 207             "id": "#",
 208             "class": "CommandLineTool"
 209         })
 210
 211         loadingContext, runtimeContext = self.helper(runner)
 212         runtimeContext.name = "test_resource_requirements"
 213
 214         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 215         arvtool.formatgraph = None
 216         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 217             j.run(runtimeContext)
 218
 219         call_args, call_kwargs = runner.api.container_requests().create.call_args
 220
 221         call_body_expected = {
 222             'environment': {
 223                 'HOME': '/var/spool/cwl',
 224                 'TMPDIR': '/tmp'
 225             },
 226             'name': 'test_resource_requirements',
 227             'runtime_constraints': {
 228                 'vcpus': 3,
 229                 'ram': 3145728000,
 230                 'keep_cache_ram': 536870912,
 231                 'API': True
 232             },
 233             'use_existing': False,
 234             'priority': 500,
 235             'mounts': {
 236                 '/tmp': {'kind': 'tmp',
 237                          "capacity": 4194304000 },
 238                 '/var/spool/cwl': {'kind': 'tmp',
 239                                    "capacity": 5242880000 }
 240             },
 241             'state': 'Committed',
 242             'output_name': 'Output for step test_resource_requirements',
 243             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 244             'output_path': '/var/spool/cwl',
 245             'output_ttl': 7200,
 246             'container_image': '99999999999999999999999999999993+99',
 247             'command': ['ls'],
 248             'cwd': '/var/spool/cwl',
 249             'scheduling_parameters': {
 250                 'partitions': ['blurb']
 251             },
 252             'properties': {},
 253             'secret_mounts': {}
 254         }
 255
 256         call_body = call_kwargs.get('body', None)
 257         self.assertNotEqual(None, call_body)
 258         for key in call_body:
 259             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 260
 261
 262     # The test passes some fields in builder.resources
 263     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 264     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 265     @mock.patch("arvados.collection.Collection")
 266     def test_initial_work_dir(self, collection_mock, keepdocker):
 267         arv_docker_clear_cache()
 268         runner = mock.MagicMock()
 269         runner.ignore_docker_for_reuse = False
 270         runner.intermediate_output_ttl = 0
 271         runner.secret_store = cwltool.secrets.SecretStore()
 272
 273         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 274         runner.api.collections().get().execute.return_value = {
 275             "portable_data_hash": "99999999999999999999999999999993+99"}
 276
 277         sourcemock = mock.MagicMock()
 278         def get_collection_mock(p):
 279             if "/" in p:
 280                 return (sourcemock, p.split("/", 1)[1])
 281             else:
 282                 return (sourcemock, "")
 283         runner.fs_access.get_collection.side_effect = get_collection_mock
 284
 285         vwdmock = mock.MagicMock()
 286         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 287
 288         tool = cmap({
 289             "inputs": [],
 290             "outputs": [],
 291             "hints": [{
 292                 "class": "InitialWorkDirRequirement",
 293                 "listing": [{
 294                     "class": "File",
 295                     "basename": "foo",
 296                     "location": "keep:99999999999999999999999999999995+99/bar"
 297                 },
 298                 {
 299                     "class": "Directory",
 300                     "basename": "foo2",
 301                     "location": "keep:99999999999999999999999999999995+99"
 302                 },
 303                 {
 304                     "class": "File",
 305                     "basename": "filename",
 306                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 307                 },
 308                 {
 309                     "class": "Directory",
 310                     "basename": "subdir",
 311                     "location": "keep:99999999999999999999999999999995+99/subdir"
 312                 }                        ]
 313             }],
 314             "baseCommand": "ls",
 315             "id": "#",
 316             "class": "CommandLineTool"
 317         })
 318
 319         loadingContext, runtimeContext = self.helper(runner)
 320         runtimeContext.name = "test_initial_work_dir"
 321
 322         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 323         arvtool.formatgraph = None
 324         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 325             j.run(runtimeContext)
 326
 327         call_args, call_kwargs = runner.api.container_requests().create.call_args
 328
 329         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 330         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 332         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 333
 334         call_body_expected = {
 335             'environment': {
 336                 'HOME': '/var/spool/cwl',
 337                 'TMPDIR': '/tmp'
 338             },
 339             'name': 'test_initial_work_dir',
 340             'runtime_constraints': {
 341                 'vcpus': 1,
 342                 'ram': 1073741824
 343             },
 344             'use_existing': True,
 345             'priority': 500,
 346             'mounts': {
 347                 '/tmp': {'kind': 'tmp',
 348                          "capacity": 1073741824 },
 349                 '/var/spool/cwl': {'kind': 'tmp',
 350                                    "capacity": 1073741824 },
 351                 '/var/spool/cwl/foo': {
 352                     'kind': 'collection',
 353                     'path': 'foo',
 354                     'portable_data_hash': '99999999999999999999999999999996+99'
 355                 },
 356                 '/var/spool/cwl/foo2': {
 357                     'kind': 'collection',
 358                     'path': 'foo2',
 359                     'portable_data_hash': '99999999999999999999999999999996+99'
 360                 },
 361                 '/var/spool/cwl/filename': {
 362                     'kind': 'collection',
 363                     'path': 'filename',
 364                     'portable_data_hash': '99999999999999999999999999999996+99'
 365                 },
 366                 '/var/spool/cwl/subdir': {
 367                     'kind': 'collection',
 368                     'path': 'subdir',
 369                     'portable_data_hash': '99999999999999999999999999999996+99'
 370                 }
 371             },
 372             'state': 'Committed',
 373             'output_name': 'Output for step test_initial_work_dir',
 374             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 375             'output_path': '/var/spool/cwl',
 376             'output_ttl': 0,
 377             'container_image': '99999999999999999999999999999993+99',
 378             'command': ['ls'],
 379             'cwd': '/var/spool/cwl',
 380             'scheduling_parameters': {
 381             },
 382             'properties': {},
 383             'secret_mounts': {}
 384         }
 385
 386         call_body = call_kwargs.get('body', None)
 387         self.assertNotEqual(None, call_body)
 388         for key in call_body:
 389             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 390
 391
 392     # Test redirecting stdin/stdout/stderr
 393     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 394     def test_redirects(self, keepdocker):
 395         arv_docker_clear_cache()
 396
 397         runner = mock.MagicMock()
 398         runner.ignore_docker_for_reuse = False
 399         runner.intermediate_output_ttl = 0
 400         runner.secret_store = cwltool.secrets.SecretStore()
 401
 402         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 403         runner.api.collections().get().execute.return_value = {
 404             "portable_data_hash": "99999999999999999999999999999993+99"}
 405
 406         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 407
 408         tool = cmap({
 409             "inputs": [],
 410             "outputs": [],
 411             "baseCommand": "ls",
 412             "stdout": "stdout.txt",
 413             "stderr": "stderr.txt",
 414             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 415             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 416             "id": "#",
 417             "class": "CommandLineTool"
 418         })
 419
 420         loadingContext, runtimeContext = self.helper(runner)
 421         runtimeContext.name = "test_run_redirect"
 422
 423         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 424         arvtool.formatgraph = None
 425         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 426             j.run(runtimeContext)
 427             runner.api.container_requests().create.assert_called_with(
 428                 body=JsonDiffMatcher({
 429                     'environment': {
 430                         'HOME': '/var/spool/cwl',
 431                         'TMPDIR': '/tmp'
 432                     },
 433                     'name': 'test_run_redirect',
 434                     'runtime_constraints': {
 435                         'vcpus': 1,
 436                         'ram': 1073741824
 437                     },
 438                     'use_existing': True,
 439                     'priority': 500,
 440                     'mounts': {
 441                         '/tmp': {'kind': 'tmp',
 442                                  "capacity": 1073741824 },
 443                         '/var/spool/cwl': {'kind': 'tmp',
 444                                            "capacity": 1073741824 },
 445                         "stderr": {
 446                             "kind": "file",
 447                             "path": "/var/spool/cwl/stderr.txt"
 448                         },
 449                         "stdin": {
 450                             "kind": "collection",
 451                             "path": "file.txt",
 452                             "portable_data_hash": "99999999999999999999999999999996+99"
 453                         },
 454                         "stdout": {
 455                             "kind": "file",
 456                             "path": "/var/spool/cwl/stdout.txt"
 457                         },
 458                     },
 459                     'state': 'Committed',
 460                     "output_name": "Output for step test_run_redirect",
 461                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 462                     'output_path': '/var/spool/cwl',
 463                     'output_ttl': 0,
 464                     'container_image': '99999999999999999999999999999993+99',
 465                     'command': ['ls', '/var/spool/cwl'],
 466                     'cwd': '/var/spool/cwl',
 467                     'scheduling_parameters': {},
 468                     'properties': {},
 469                     'secret_mounts': {}
 470                 }))
 471
 472     @mock.patch("arvados.collection.Collection")
 473     def test_done(self, col):
 474         api = mock.MagicMock()
 475
 476         runner = mock.MagicMock()
 477         runner.api = api
 478         runner.num_retries = 0
 479         runner.ignore_docker_for_reuse = False
 480         runner.intermediate_output_ttl = 0
 481         runner.secret_store = cwltool.secrets.SecretStore()
 482
 483         runner.api.containers().get().execute.return_value = {"state":"Complete",
 484                                                               "output": "abc+123",
 485                                                               "exit_code": 0}
 486
 487         col().open.return_value = []
 488
 489         loadingContext, runtimeContext = self.helper(runner)
 490
 491         arvjob = arvados_cwl.ArvadosContainer(runner,
 492                                               runtimeContext,
 493                                               mock.MagicMock(),
 494                                               {},
 495                                               None,
 496                                               [],
 497                                               [],
 498                                               "testjob")
 499         arvjob.output_callback = mock.MagicMock()
 500         arvjob.collect_outputs = mock.MagicMock()
 501         arvjob.successCodes = [0]
 502         arvjob.outdir = "/var/spool/cwl"
 503         arvjob.output_ttl = 3600
 504
 505         arvjob.collect_outputs.return_value = {"out": "stuff"}
 506
 507         arvjob.done({
 508             "state": "Final",
 509             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 510             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 511             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 512             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 513             "modified_at": "2017-05-26T12:01:22Z"
 514         })
 515
 516         self.assertFalse(api.collections().create.called)
 517         self.assertFalse(runner.runtime_status_error.called)
 518
 519         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 520         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 521         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 522
 523     # Test to make sure we dont call runtime_status_update if we already did
 524     # some where higher up in the call stack
 525     @mock.patch("arvados_cwl.util.get_current_container")
 526     def test_recursive_runtime_status_update(self, gcc_mock):
 527         self.setup_and_test_container_executor_and_logging(gcc_mock)
 528         root_logger = logging.getLogger('')
 529
 530         # get_current_container is invoked when we call runtime_status_update
 531         # so try and log again!
 532         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 533         try:
 534             root_logger.error("First Error")
 535         except RuntimeError:
 536             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 537
 538
 539     # Test to make sure that an exception raised from
 540     # get_current_container doesn't cause the logger to raise an
 541     # exception
 542     @mock.patch("arvados_cwl.util.get_current_container")
 543     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 544         self.setup_and_test_container_executor_and_logging(gcc_mock)
 545         root_logger = logging.getLogger('')
 546
 547         # get_current_container is invoked when we call
 548         # runtime_status_update, it is going to also raise an
 549         # exception.
 550         gcc_mock.side_effect = Exception("Second Error")
 551         try:
 552             root_logger.error("First Error")
 553         except Exception:
 554             self.fail("Exception in logger should not propagate")
 555         self.assertTrue(gcc_mock.called)
 556
 557     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 558     @mock.patch("arvados_cwl.util.get_current_container")
 559     @mock.patch("arvados.collection.CollectionReader")
 560     @mock.patch("arvados.collection.Collection")
 561     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 562         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 563
 564         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 565         self.assertTrue(gcc_mock.called)
 566
 567         runner.num_retries = 0
 568         runner.ignore_docker_for_reuse = False
 569         runner.intermediate_output_ttl = 0
 570         runner.secret_store = cwltool.secrets.SecretStore()
 571         runner.label = mock.MagicMock()
 572         runner.label.return_value = '[container testjob]'
 573
 574         runner.api.containers().get().execute.return_value = {
 575             "state":"Complete",
 576             "output": "abc+123",
 577             "exit_code": 1,
 578             "log": "def+234"
 579         }
 580
 581         col().open.return_value = []
 582
 583         loadingContext, runtimeContext = self.helper(runner)
 584
 585         arvjob = arvados_cwl.ArvadosContainer(runner,
 586                                               runtimeContext,
 587                                               mock.MagicMock(),
 588                                               {},
 589                                               None,
 590                                               [],
 591                                               [],
 592                                               "testjob")
 593         arvjob.output_callback = mock.MagicMock()
 594         arvjob.collect_outputs = mock.MagicMock()
 595         arvjob.successCodes = [0]
 596         arvjob.outdir = "/var/spool/cwl"
 597         arvjob.output_ttl = 3600
 598         arvjob.collect_outputs.return_value = {"out": "stuff"}
 599
 600         arvjob.done({
 601             "state": "Final",
 602             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 603             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 604             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 605             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 606             "modified_at": "2017-05-26T12:01:22Z"
 607         })
 608
 609         rts_mock.assert_called_with(
 610             'error',
 611             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 612             '  ** log is empty **'
 613         )
 614         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 615
 616     # The test passes no builder.resources
 617     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 618     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 619     def test_mounts(self, keepdocker):
 620         arv_docker_clear_cache()
 621
 622         runner = mock.MagicMock()
 623         runner.ignore_docker_for_reuse = False
 624         runner.intermediate_output_ttl = 0
 625         runner.secret_store = cwltool.secrets.SecretStore()
 626
 627         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 628         runner.api.collections().get().execute.return_value = {
 629             "portable_data_hash": "99999999999999999999999999999994+99",
 630             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 631
 632         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 633
 634         tool = cmap({
 635             "inputs": [
 636                 {"id": "p1",
 637                  "type": "Directory"}
 638             ],
 639             "outputs": [],
 640             "baseCommand": "ls",
 641             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 642             "id": "#",
 643             "class": "CommandLineTool"
 644         })
 645
 646         loadingContext, runtimeContext = self.helper(runner)
 647         runtimeContext.name = "test_run_mounts"
 648
 649         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 650         arvtool.formatgraph = None
 651         job_order = {
 652             "p1": {
 653                 "class": "Directory",
 654                 "location": "keep:99999999999999999999999999999994+44",
 655                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 656                 "listing": [
 657                     {
 658                         "class": "File",
 659                         "location": "keep:99999999999999999999999999999994+44/file1",
 660                     },
 661                     {
 662                         "class": "File",
 663                         "location": "keep:99999999999999999999999999999994+44/file2",
 664                     }
 665                 ]
 666             }
 667         }
 668         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 669             j.run(runtimeContext)
 670             runner.api.container_requests().create.assert_called_with(
 671                 body=JsonDiffMatcher({
 672                     'environment': {
 673                         'HOME': '/var/spool/cwl',
 674                         'TMPDIR': '/tmp'
 675                     },
 676                     'name': 'test_run_mounts',
 677                     'runtime_constraints': {
 678                         'vcpus': 1,
 679                         'ram': 1073741824
 680                     },
 681                     'use_existing': True,
 682                     'priority': 500,
 683                     'mounts': {
 684                         "/keep/99999999999999999999999999999994+44": {
 685                             "kind": "collection",
 686                             "portable_data_hash": "99999999999999999999999999999994+44",
 687                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 688                         },
 689                         '/tmp': {'kind': 'tmp',
 690                                  "capacity": 1073741824 },
 691                         '/var/spool/cwl': {'kind': 'tmp',
 692                                            "capacity": 1073741824 }
 693                     },
 694                     'state': 'Committed',
 695                     'output_name': 'Output for step test_run_mounts',
 696                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 697                     'output_path': '/var/spool/cwl',
 698                     'output_ttl': 0,
 699                     'container_image': '99999999999999999999999999999994+99',
 700                     'command': ['ls', '/var/spool/cwl'],
 701                     'cwd': '/var/spool/cwl',
 702                     'scheduling_parameters': {},
 703                     'properties': {},
 704                     'secret_mounts': {}
 705                 }))
 706
 707     # The test passes no builder.resources
 708     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 709     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 710     def test_secrets(self, keepdocker):
 711         arv_docker_clear_cache()
 712
 713         runner = mock.MagicMock()
 714         runner.ignore_docker_for_reuse = False
 715         runner.intermediate_output_ttl = 0
 716         runner.secret_store = cwltool.secrets.SecretStore()
 717
 718         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 719         runner.api.collections().get().execute.return_value = {
 720             "portable_data_hash": "99999999999999999999999999999993+99"}
 721
 722         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 723
 724         tool = cmap({"arguments": ["md5sum", "example.conf"],
 725                      "class": "CommandLineTool",
 726                      "hints": [
 727                          {
 728                              "class": "http://commonwl.org/cwltool#Secrets",
 729                              "secrets": [
 730                                  "#secret_job.cwl/pw"
 731                              ]
 732                          }
 733                      ],
 734                      "id": "#secret_job.cwl",
 735                      "inputs": [
 736                          {
 737                              "id": "#secret_job.cwl/pw",
 738                              "type": "string"
 739                          }
 740                      ],
 741                      "outputs": [
 742                      ],
 743                      "requirements": [
 744                          {
 745                              "class": "InitialWorkDirRequirement",
 746                              "listing": [
 747                                  {
 748                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 749                                      "entryname": "example.conf"
 750                                  }
 751                              ]
 752                          }
 753                      ]})
 754
 755         loadingContext, runtimeContext = self.helper(runner)
 756         runtimeContext.name = "test_secrets"
 757
 758         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 759         arvtool.formatgraph = None
 760
 761         job_order = {"pw": "blorp"}
 762         runner.secret_store.store(["pw"], job_order)
 763
 764         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 765             j.run(runtimeContext)
 766             runner.api.container_requests().create.assert_called_with(
 767                 body=JsonDiffMatcher({
 768                     'environment': {
 769                         'HOME': '/var/spool/cwl',
 770                         'TMPDIR': '/tmp'
 771                     },
 772                     'name': 'test_secrets',
 773                     'runtime_constraints': {
 774                         'vcpus': 1,
 775                         'ram': 1073741824
 776                     },
 777                     'use_existing': True,
 778                     'priority': 500,
 779                     'mounts': {
 780                         '/tmp': {'kind': 'tmp',
 781                                  "capacity": 1073741824
 782                              },
 783                         '/var/spool/cwl': {'kind': 'tmp',
 784                                            "capacity": 1073741824 }
 785                     },
 786                     'state': 'Committed',
 787                     'output_name': 'Output for step test_secrets',
 788                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 789                     'output_path': '/var/spool/cwl',
 790                     'output_ttl': 0,
 791                     'container_image': '99999999999999999999999999999993+99',
 792                     'command': ['md5sum', 'example.conf'],
 793                     'cwd': '/var/spool/cwl',
 794                     'scheduling_parameters': {},
 795                     'properties': {},
 796                     "secret_mounts": {
 797                         "/var/spool/cwl/example.conf": {
 798                             "content": "username: user\npassword: blorp\n",
 799                             "kind": "text"
 800                         }
 801                     }
 802                 }))
 803
 804     # The test passes no builder.resources
 805     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 806     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 807     def test_timelimit(self, keepdocker):
 808         arv_docker_clear_cache()
 809
 810         runner = mock.MagicMock()
 811         runner.ignore_docker_for_reuse = False
 812         runner.intermediate_output_ttl = 0
 813         runner.secret_store = cwltool.secrets.SecretStore()
 814
 815         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 816         runner.api.collections().get().execute.return_value = {
 817             "portable_data_hash": "99999999999999999999999999999993+99"}
 818
 819         tool = cmap({
 820             "inputs": [],
 821             "outputs": [],
 822             "baseCommand": "ls",
 823             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 824             "id": "#",
 825             "class": "CommandLineTool",
 826             "hints": [
 827                 {
 828                     "class": "ToolTimeLimit",
 829                     "timelimit": 42
 830                 }
 831             ]
 832         })
 833
 834         loadingContext, runtimeContext = self.helper(runner)
 835         runtimeContext.name = "test_timelimit"
 836
 837         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 838         arvtool.formatgraph = None
 839
 840         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 841             j.run(runtimeContext)
 842
 843         _, kwargs = runner.api.container_requests().create.call_args
 844         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 845
 846
 847 class TestWorkflow(unittest.TestCase):
 848     def setUp(self):
 849         cwltool.process._names = set()
 850
 851     def helper(self, runner, enable_reuse=True):
 852         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 853
 854         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 855                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 856
 857         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 858         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 859         document_loader.fetch_text = document_loader.fetcher.fetch_text
 860         document_loader.check_exists = document_loader.fetcher.check_exists
 861
 862         loadingContext = arvados_cwl.context.ArvLoadingContext(
 863             {"avsc_names": avsc_names,
 864              "basedir": "",
 865              "make_fs_access": make_fs_access,
 866              "loader": document_loader,
 867              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 868              "construct_tool_object": runner.arv_make_tool})
 869         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 870             {"work_api": "containers",
 871              "basedir": "",
 872              "name": "test_run_wf_"+str(enable_reuse),
 873              "make_fs_access": make_fs_access,
 874              "tmpdir": "/tmp",
 875              "enable_reuse": enable_reuse,
 876              "priority": 500})
 877
 878         return loadingContext, runtimeContext
 879
 880     # The test passes no builder.resources
 881     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 882     @mock.patch("arvados.collection.CollectionReader")
 883     @mock.patch("arvados.collection.Collection")
 884     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 885     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 886         arv_docker_clear_cache()
 887         arvados_cwl.add_arv_hints()
 888
 889         api = mock.MagicMock()
 890         api._rootDesc = get_rootDesc()
 891
 892         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 893         self.assertEqual(runner.work_api, 'containers')
 894
 895         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 896         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 897         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 898                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 899
 900         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 901         runner.ignore_docker_for_reuse = False
 902         runner.num_retries = 0
 903         runner.secret_store = cwltool.secrets.SecretStore()
 904
 905         loadingContext, runtimeContext = self.helper(runner)
 906         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 907
 908         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 909         metadata["cwlVersion"] = tool["cwlVersion"]
 910
 911         mockc = mock.MagicMock()
 912         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 913         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 914
 915         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 916         arvtool.formatgraph = None
 917         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 918
 919         next(it).run(runtimeContext)
 920         next(it).run(runtimeContext)
 921
 922         with open("tests/wf/scatter2_subwf.cwl") as f:
 923             subwf = StripYAMLComments(f.read()).rstrip()
 924
 925         runner.api.container_requests().create.assert_called_with(
 926             body=JsonDiffMatcher({
 927                 "command": [
 928                     "cwltool",
 929                     "--no-container",
 930                     "--move-outputs",
 931                     "--preserve-entire-environment",
 932                     "workflow.cwl",
 933                     "cwl.input.yml"
 934                 ],
 935                 "container_image": "99999999999999999999999999999993+99",
 936                 "cwd": "/var/spool/cwl",
 937                 "environment": {
 938                     "HOME": "/var/spool/cwl",
 939                     "TMPDIR": "/tmp"
 940                 },
 941                 "mounts": {
 942                     "/keep/99999999999999999999999999999999+118": {
 943                         "kind": "collection",
 944                         "portable_data_hash": "99999999999999999999999999999999+118"
 945                     },
 946                     "/tmp": {
 947                         "capacity": 1073741824,
 948                         "kind": "tmp"
 949                     },
 950                     "/var/spool/cwl": {
 951                         "capacity": 1073741824,
 952                         "kind": "tmp"
 953                     },
 954                     "/var/spool/cwl/cwl.input.yml": {
 955                         "kind": "collection",
 956                         "path": "cwl.input.yml",
 957                         "portable_data_hash": "99999999999999999999999999999996+99"
 958                     },
 959                     "/var/spool/cwl/workflow.cwl": {
 960                         "kind": "collection",
 961                         "path": "workflow.cwl",
 962                         "portable_data_hash": "99999999999999999999999999999996+99"
 963                     },
 964                     "stdout": {
 965                         "kind": "file",
 966                         "path": "/var/spool/cwl/cwl.output.json"
 967                     }
 968                 },
 969                 "name": "scatterstep",
 970                 "output_name": "Output for step scatterstep",
 971                 "output_path": "/var/spool/cwl",
 972                 "output_ttl": 0,
 973                 "priority": 500,
 974                 "properties": {},
 975                 "runtime_constraints": {
 976                     "ram": 1073741824,
 977                     "vcpus": 1
 978                 },
 979                 "scheduling_parameters": {},
 980                 "secret_mounts": {},
 981                 "state": "Committed",
 982                 "use_existing": True
 983             }))
 984         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 985         mockc.open().__enter__().write.assert_has_calls([mock.call(
 986 '''{
 987   "fileblub": {
 988     "basename": "token.txt",
 989     "class": "File",
 990     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 991     "size": 0
 992   },
 993   "sleeptime": 5
 994 }''')])
 995
 996     # The test passes no builder.resources
 997     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 998     @mock.patch("arvados.collection.CollectionReader")
 999     @mock.patch("arvados.collection.Collection")
1000     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1001     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1002         arv_docker_clear_cache()
1003         arvados_cwl.add_arv_hints()
1004
1005         api = mock.MagicMock()
1006         api._rootDesc = get_rootDesc()
1007
1008         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1009         self.assertEqual(runner.work_api, 'containers')
1010
1011         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1012         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1013                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1014         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1015                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1016
1017         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1018         runner.ignore_docker_for_reuse = False
1019         runner.num_retries = 0
1020         runner.secret_store = cwltool.secrets.SecretStore()
1021
1022         loadingContext, runtimeContext = self.helper(runner)
1023         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1024         loadingContext.do_update = True
1025         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1026
1027         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1028
1029         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1030         arvtool.formatgraph = None
1031         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1032
1033         next(it).run(runtimeContext)
1034         next(it).run(runtimeContext)
1035
1036         with open("tests/wf/echo-subwf.cwl") as f:
1037             subwf = StripYAMLComments(f.read())
1038
1039         runner.api.container_requests().create.assert_called_with(
1040             body=JsonDiffMatcher({
1041                 'output_ttl': 0,
1042                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1043                 'scheduling_parameters': {},
1044                 'name': u'echo-subwf',
1045                 'secret_mounts': {},
1046                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1047                 'properties': {},
1048                 'priority': 500,
1049                 'mounts': {
1050                     '/var/spool/cwl/cwl.input.yml': {
1051                         'portable_data_hash': '99999999999999999999999999999996+99',
1052                         'kind': 'collection',
1053                         'path': 'cwl.input.yml'
1054                     },
1055                     '/var/spool/cwl/workflow.cwl': {
1056                         'portable_data_hash': '99999999999999999999999999999996+99',
1057                         'kind': 'collection',
1058                         'path': 'workflow.cwl'
1059                     },
1060                     'stdout': {
1061                         'path': '/var/spool/cwl/cwl.output.json',
1062                         'kind': 'file'
1063                     },
1064                     '/tmp': {
1065                         'kind': 'tmp',
1066                         'capacity': 1073741824
1067                     }, '/var/spool/cwl': {
1068                         'kind': 'tmp',
1069                         'capacity': 3221225472
1070                     }
1071                 },
1072                 'state': 'Committed',
1073                 'output_path': '/var/spool/cwl',
1074                 'container_image': '99999999999999999999999999999993+99',
1075                 'command': [
1076                     u'cwltool',
1077                     u'--no-container',
1078                     u'--move-outputs',
1079                     u'--preserve-entire-environment',
1080                     u'workflow.cwl',
1081                     u'cwl.input.yml'
1082                 ],
1083                 'use_existing': True,
1084                 'output_name': u'Output for step echo-subwf',
1085                 'cwd': '/var/spool/cwl'
1086             }))
1087
1088     def test_default_work_api(self):
1089         arvados_cwl.add_arv_hints()
1090
1091         api = mock.MagicMock()
1092         api._rootDesc = copy.deepcopy(get_rootDesc())
1093         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1094         self.assertEqual(runner.work_api, 'containers')