sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 from schema_salad.ref_resolver import Loader
  22 from schema_salad.sourceline import cmap
  23
  24 from .matcher import JsonDiffMatcher, StripYAMLComments
  25 from .mock_discovery import get_rootDesc
  26
  27 if not os.getenv('ARVADOS_DEBUG'):
  28     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  29     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  30
  31 class CollectionMock(object):
  32     def __init__(self, vwdmock, *args, **kwargs):
  33         self.vwdmock = vwdmock
  34         self.count = 0
  35
  36     def open(self, *args, **kwargs):
  37         self.count += 1
  38         return self.vwdmock.open(*args, **kwargs)
  39
  40     def copy(self, *args, **kwargs):
  41         self.count += 1
  42         self.vwdmock.copy(*args, **kwargs)
  43
  44     def save_new(self, *args, **kwargs):
  45         pass
  46
  47     def __len__(self):
  48         return self.count
  49
  50     def portable_data_hash(self):
  51         if self.count == 0:
  52             return arvados.config.EMPTY_BLOCK_LOCATOR
  53         else:
  54             return "99999999999999999999999999999996+99"
  55
  56
  57 class TestContainer(unittest.TestCase):
  58
  59     def setUp(self):
  60         cwltool.process._names = set()
  61
  62     def helper(self, runner, enable_reuse=True):
  63         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
  64
  65         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  66                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  67         loadingContext = arvados_cwl.context.ArvLoadingContext(
  68             {"avsc_names": avsc_names,
  69              "basedir": "",
  70              "make_fs_access": make_fs_access,
  71              "loader": Loader({}),
  72              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
  73         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  74             {"work_api": "containers",
  75              "basedir": "",
  76              "name": "test_run_"+str(enable_reuse),
  77              "make_fs_access": make_fs_access,
  78              "tmpdir": "/tmp",
  79              "enable_reuse": enable_reuse,
  80              "priority": 500,
  81              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82             })
  83
  84         return loadingContext, runtimeContext
  85
  86     # Helper function to set up the ArvCwlExecutor to use the containers api
  87     # and test that the RuntimeStatusLoggingHandler is set up correctly
  88     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
  89         api = mock.MagicMock()
  90         api._rootDesc = copy.deepcopy(get_rootDesc())
  91
  92         # Make sure ArvCwlExecutor thinks it's running inside a container so it
  93         # adds the logging handler that will call runtime_status_update() mock
  94         self.assertFalse(gcc_mock.called)
  95         runner = arvados_cwl.ArvCwlExecutor(api)
  96         self.assertEqual(runner.work_api, 'containers')
  97         root_logger = logging.getLogger('')
  98         handlerClasses = [h.__class__ for h in root_logger.handlers]
  99         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 100         return runner
 101
 102     # The test passes no builder.resources
 103     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 104     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 105     def test_run(self, keepdocker):
 106         for enable_reuse in (True, False):
 107             arv_docker_clear_cache()
 108
 109             runner = mock.MagicMock()
 110             runner.ignore_docker_for_reuse = False
 111             runner.intermediate_output_ttl = 0
 112             runner.secret_store = cwltool.secrets.SecretStore()
 113
 114             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 115             runner.api.collections().get().execute.return_value = {
 116                 "portable_data_hash": "99999999999999999999999999999993+99"}
 117
 118             tool = cmap({
 119                 "inputs": [],
 120                 "outputs": [],
 121                 "baseCommand": "ls",
 122                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 123                 "id": "#",
 124                 "class": "CommandLineTool"
 125             })
 126
 127             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 128
 129             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 130             arvtool.formatgraph = None
 131
 132             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 133                 j.run(runtimeContext)
 134                 runner.api.container_requests().create.assert_called_with(
 135                     body=JsonDiffMatcher({
 136                         'environment': {
 137                             'HOME': '/var/spool/cwl',
 138                             'TMPDIR': '/tmp'
 139                         },
 140                         'name': 'test_run_'+str(enable_reuse),
 141                         'runtime_constraints': {
 142                             'vcpus': 1,
 143                             'ram': 1073741824
 144                         },
 145                         'use_existing': enable_reuse,
 146                         'priority': 500,
 147                         'mounts': {
 148                             '/tmp': {'kind': 'tmp',
 149                                      "capacity": 1073741824
 150                                  },
 151                             '/var/spool/cwl': {'kind': 'tmp',
 152                                                "capacity": 1073741824 }
 153                         },
 154                         'state': 'Committed',
 155                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 156                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 157                         'output_path': '/var/spool/cwl',
 158                         'output_ttl': 0,
 159                         'container_image': '99999999999999999999999999999993+99',
 160                         'command': ['ls', '/var/spool/cwl'],
 161                         'cwd': '/var/spool/cwl',
 162                         'scheduling_parameters': {},
 163                         'properties': {},
 164                         'secret_mounts': {}
 165                     }))
 166
 167     # The test passes some fields in builder.resources
 168     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 169     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 170     def test_resource_requirements(self, keepdocker):
 171         arv_docker_clear_cache()
 172         runner = mock.MagicMock()
 173         runner.ignore_docker_for_reuse = False
 174         runner.intermediate_output_ttl = 3600
 175         runner.secret_store = cwltool.secrets.SecretStore()
 176
 177         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 178         runner.api.collections().get().execute.return_value = {
 179             "portable_data_hash": "99999999999999999999999999999993+99"}
 180
 181         tool = cmap({
 182             "inputs": [],
 183             "outputs": [],
 184             "hints": [{
 185                 "class": "ResourceRequirement",
 186                 "coresMin": 3,
 187                 "ramMin": 3000,
 188                 "tmpdirMin": 4000,
 189                 "outdirMin": 5000
 190             }, {
 191                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 192                 "keep_cache": 512
 193             }, {
 194                 "class": "http://arvados.org/cwl#APIRequirement",
 195             }, {
 196                 "class": "http://arvados.org/cwl#PartitionRequirement",
 197                 "partition": "blurb"
 198             }, {
 199                 "class": "http://arvados.org/cwl#IntermediateOutput",
 200                 "outputTTL": 7200
 201             }, {
 202                 "class": "http://arvados.org/cwl#ReuseRequirement",
 203                 "enableReuse": False
 204             }],
 205             "baseCommand": "ls",
 206             "id": "#",
 207             "class": "CommandLineTool"
 208         })
 209
 210         loadingContext, runtimeContext = self.helper(runner)
 211         runtimeContext.name = "test_resource_requirements"
 212
 213         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 214         arvtool.formatgraph = None
 215         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 216             j.run(runtimeContext)
 217
 218         call_args, call_kwargs = runner.api.container_requests().create.call_args
 219
 220         call_body_expected = {
 221             'environment': {
 222                 'HOME': '/var/spool/cwl',
 223                 'TMPDIR': '/tmp'
 224             },
 225             'name': 'test_resource_requirements',
 226             'runtime_constraints': {
 227                 'vcpus': 3,
 228                 'ram': 3145728000,
 229                 'keep_cache_ram': 536870912,
 230                 'API': True
 231             },
 232             'use_existing': False,
 233             'priority': 500,
 234             'mounts': {
 235                 '/tmp': {'kind': 'tmp',
 236                          "capacity": 4194304000 },
 237                 '/var/spool/cwl': {'kind': 'tmp',
 238                                    "capacity": 5242880000 }
 239             },
 240             'state': 'Committed',
 241             'output_name': 'Output for step test_resource_requirements',
 242             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 243             'output_path': '/var/spool/cwl',
 244             'output_ttl': 7200,
 245             'container_image': '99999999999999999999999999999993+99',
 246             'command': ['ls'],
 247             'cwd': '/var/spool/cwl',
 248             'scheduling_parameters': {
 249                 'partitions': ['blurb']
 250             },
 251             'properties': {},
 252             'secret_mounts': {}
 253         }
 254
 255         call_body = call_kwargs.get('body', None)
 256         self.assertNotEqual(None, call_body)
 257         for key in call_body:
 258             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 259
 260
 261     # The test passes some fields in builder.resources
 262     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 263     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 264     @mock.patch("arvados.collection.Collection")
 265     def test_initial_work_dir(self, collection_mock, keepdocker):
 266         arv_docker_clear_cache()
 267         runner = mock.MagicMock()
 268         runner.ignore_docker_for_reuse = False
 269         runner.intermediate_output_ttl = 0
 270         runner.secret_store = cwltool.secrets.SecretStore()
 271
 272         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 273         runner.api.collections().get().execute.return_value = {
 274             "portable_data_hash": "99999999999999999999999999999993+99"}
 275
 276         sourcemock = mock.MagicMock()
 277         def get_collection_mock(p):
 278             if "/" in p:
 279                 return (sourcemock, p.split("/", 1)[1])
 280             else:
 281                 return (sourcemock, "")
 282         runner.fs_access.get_collection.side_effect = get_collection_mock
 283
 284         vwdmock = mock.MagicMock()
 285         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 286
 287         tool = cmap({
 288             "inputs": [],
 289             "outputs": [],
 290             "hints": [{
 291                 "class": "InitialWorkDirRequirement",
 292                 "listing": [{
 293                     "class": "File",
 294                     "basename": "foo",
 295                     "location": "keep:99999999999999999999999999999995+99/bar"
 296                 },
 297                 {
 298                     "class": "Directory",
 299                     "basename": "foo2",
 300                     "location": "keep:99999999999999999999999999999995+99"
 301                 },
 302                 {
 303                     "class": "File",
 304                     "basename": "filename",
 305                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 306                 },
 307                 {
 308                     "class": "Directory",
 309                     "basename": "subdir",
 310                     "location": "keep:99999999999999999999999999999995+99/subdir"
 311                 }                        ]
 312             }],
 313             "baseCommand": "ls",
 314             "id": "#",
 315             "class": "CommandLineTool"
 316         })
 317
 318         loadingContext, runtimeContext = self.helper(runner)
 319         runtimeContext.name = "test_initial_work_dir"
 320
 321         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 322         arvtool.formatgraph = None
 323         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 324             j.run(runtimeContext)
 325
 326         call_args, call_kwargs = runner.api.container_requests().create.call_args
 327
 328         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 329         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 330         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 331         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 332
 333         call_body_expected = {
 334             'environment': {
 335                 'HOME': '/var/spool/cwl',
 336                 'TMPDIR': '/tmp'
 337             },
 338             'name': 'test_initial_work_dir',
 339             'runtime_constraints': {
 340                 'vcpus': 1,
 341                 'ram': 1073741824
 342             },
 343             'use_existing': True,
 344             'priority': 500,
 345             'mounts': {
 346                 '/tmp': {'kind': 'tmp',
 347                          "capacity": 1073741824 },
 348                 '/var/spool/cwl': {'kind': 'tmp',
 349                                    "capacity": 1073741824 },
 350                 '/var/spool/cwl/foo': {
 351                     'kind': 'collection',
 352                     'path': 'foo',
 353                     'portable_data_hash': '99999999999999999999999999999996+99'
 354                 },
 355                 '/var/spool/cwl/foo2': {
 356                     'kind': 'collection',
 357                     'path': 'foo2',
 358                     'portable_data_hash': '99999999999999999999999999999996+99'
 359                 },
 360                 '/var/spool/cwl/filename': {
 361                     'kind': 'collection',
 362                     'path': 'filename',
 363                     'portable_data_hash': '99999999999999999999999999999996+99'
 364                 },
 365                 '/var/spool/cwl/subdir': {
 366                     'kind': 'collection',
 367                     'path': 'subdir',
 368                     'portable_data_hash': '99999999999999999999999999999996+99'
 369                 }
 370             },
 371             'state': 'Committed',
 372             'output_name': 'Output for step test_initial_work_dir',
 373             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 374             'output_path': '/var/spool/cwl',
 375             'output_ttl': 0,
 376             'container_image': '99999999999999999999999999999993+99',
 377             'command': ['ls'],
 378             'cwd': '/var/spool/cwl',
 379             'scheduling_parameters': {
 380             },
 381             'properties': {},
 382             'secret_mounts': {}
 383         }
 384
 385         call_body = call_kwargs.get('body', None)
 386         self.assertNotEqual(None, call_body)
 387         for key in call_body:
 388             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 389
 390
 391     # Test redirecting stdin/stdout/stderr
 392     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 393     def test_redirects(self, keepdocker):
 394         arv_docker_clear_cache()
 395
 396         runner = mock.MagicMock()
 397         runner.ignore_docker_for_reuse = False
 398         runner.intermediate_output_ttl = 0
 399         runner.secret_store = cwltool.secrets.SecretStore()
 400
 401         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 402         runner.api.collections().get().execute.return_value = {
 403             "portable_data_hash": "99999999999999999999999999999993+99"}
 404
 405         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 406
 407         tool = cmap({
 408             "inputs": [],
 409             "outputs": [],
 410             "baseCommand": "ls",
 411             "stdout": "stdout.txt",
 412             "stderr": "stderr.txt",
 413             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 414             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 415             "id": "#",
 416             "class": "CommandLineTool"
 417         })
 418
 419         loadingContext, runtimeContext = self.helper(runner)
 420         runtimeContext.name = "test_run_redirect"
 421
 422         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 423         arvtool.formatgraph = None
 424         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 425             j.run(runtimeContext)
 426             runner.api.container_requests().create.assert_called_with(
 427                 body=JsonDiffMatcher({
 428                     'environment': {
 429                         'HOME': '/var/spool/cwl',
 430                         'TMPDIR': '/tmp'
 431                     },
 432                     'name': 'test_run_redirect',
 433                     'runtime_constraints': {
 434                         'vcpus': 1,
 435                         'ram': 1073741824
 436                     },
 437                     'use_existing': True,
 438                     'priority': 500,
 439                     'mounts': {
 440                         '/tmp': {'kind': 'tmp',
 441                                  "capacity": 1073741824 },
 442                         '/var/spool/cwl': {'kind': 'tmp',
 443                                            "capacity": 1073741824 },
 444                         "stderr": {
 445                             "kind": "file",
 446                             "path": "/var/spool/cwl/stderr.txt"
 447                         },
 448                         "stdin": {
 449                             "kind": "collection",
 450                             "path": "file.txt",
 451                             "portable_data_hash": "99999999999999999999999999999996+99"
 452                         },
 453                         "stdout": {
 454                             "kind": "file",
 455                             "path": "/var/spool/cwl/stdout.txt"
 456                         },
 457                     },
 458                     'state': 'Committed',
 459                     "output_name": "Output for step test_run_redirect",
 460                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 461                     'output_path': '/var/spool/cwl',
 462                     'output_ttl': 0,
 463                     'container_image': '99999999999999999999999999999993+99',
 464                     'command': ['ls', '/var/spool/cwl'],
 465                     'cwd': '/var/spool/cwl',
 466                     'scheduling_parameters': {},
 467                     'properties': {},
 468                     'secret_mounts': {}
 469                 }))
 470
 471     @mock.patch("arvados.collection.Collection")
 472     def test_done(self, col):
 473         api = mock.MagicMock()
 474
 475         runner = mock.MagicMock()
 476         runner.api = api
 477         runner.num_retries = 0
 478         runner.ignore_docker_for_reuse = False
 479         runner.intermediate_output_ttl = 0
 480         runner.secret_store = cwltool.secrets.SecretStore()
 481
 482         runner.api.containers().get().execute.return_value = {"state":"Complete",
 483                                                               "output": "abc+123",
 484                                                               "exit_code": 0}
 485
 486         col().open.return_value = []
 487
 488         loadingContext, runtimeContext = self.helper(runner)
 489
 490         arvjob = arvados_cwl.ArvadosContainer(runner,
 491                                               runtimeContext,
 492                                               mock.MagicMock(),
 493                                               {},
 494                                               None,
 495                                               [],
 496                                               [],
 497                                               "testjob")
 498         arvjob.output_callback = mock.MagicMock()
 499         arvjob.collect_outputs = mock.MagicMock()
 500         arvjob.successCodes = [0]
 501         arvjob.outdir = "/var/spool/cwl"
 502         arvjob.output_ttl = 3600
 503
 504         arvjob.collect_outputs.return_value = {"out": "stuff"}
 505
 506         arvjob.done({
 507             "state": "Final",
 508             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 509             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 510             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 511             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 512             "modified_at": "2017-05-26T12:01:22Z"
 513         })
 514
 515         self.assertFalse(api.collections().create.called)
 516         self.assertFalse(runner.runtime_status_error.called)
 517
 518         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 519         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 520         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 521
 522     # Test to make sure we dont call runtime_status_update if we already did
 523     # some where higher up in the call stack
 524     @mock.patch("arvados_cwl.util.get_current_container")
 525     def test_recursive_runtime_status_update(self, gcc_mock):
 526         self.setup_and_test_container_executor_and_logging(gcc_mock)
 527         root_logger = logging.getLogger('')
 528
 529         # get_current_container is invoked when we call runtime_status_update
 530         # so try and log again!
 531         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 532         try:
 533             root_logger.error("First Error")
 534         except RuntimeError:
 535             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 536
 537
 538     # Test to make sure that an exception raised from get_current_container doesn't
 539     @mock.patch("arvados_cwl.util.get_current_container")
 540     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 541         self.setup_and_test_container_executor_and_logging(gcc_mock)
 542         root_logger = logging.getLogger('')
 543
 544         # get_current_container is invoked when we call runtime_status_update
 545         # so try and log again!
 546         gcc_mock.side_effect = Exception("Second Error")
 547         try:
 548             root_logger.error("First Error")
 549         except RuntimeError:
 550             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 551
 552     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 553     @mock.patch("arvados_cwl.util.get_current_container")
 554     @mock.patch("arvados.collection.CollectionReader")
 555     @mock.patch("arvados.collection.Collection")
 556     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 557         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 558
 559         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 560         self.assertTrue(gcc_mock.called)
 561
 562         runner.num_retries = 0
 563         runner.ignore_docker_for_reuse = False
 564         runner.intermediate_output_ttl = 0
 565         runner.secret_store = cwltool.secrets.SecretStore()
 566         runner.label = mock.MagicMock()
 567         runner.label.return_value = '[container testjob]'
 568
 569         runner.api.containers().get().execute.return_value = {
 570             "state":"Complete",
 571             "output": "abc+123",
 572             "exit_code": 1,
 573             "log": "def+234"
 574         }
 575
 576         col().open.return_value = []
 577
 578         loadingContext, runtimeContext = self.helper(runner)
 579
 580         arvjob = arvados_cwl.ArvadosContainer(runner,
 581                                               runtimeContext,
 582                                               mock.MagicMock(),
 583                                               {},
 584                                               None,
 585                                               [],
 586                                               [],
 587                                               "testjob")
 588         arvjob.output_callback = mock.MagicMock()
 589         arvjob.collect_outputs = mock.MagicMock()
 590         arvjob.successCodes = [0]
 591         arvjob.outdir = "/var/spool/cwl"
 592         arvjob.output_ttl = 3600
 593         arvjob.collect_outputs.return_value = {"out": "stuff"}
 594
 595         arvjob.done({
 596             "state": "Final",
 597             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 598             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 599             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 600             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 601             "modified_at": "2017-05-26T12:01:22Z"
 602         })
 603
 604         rts_mock.assert_called_with(
 605             'error',
 606             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 607             '  ** log is empty **'
 608         )
 609         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 610
 611     # The test passes no builder.resources
 612     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 613     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 614     def test_mounts(self, keepdocker):
 615         arv_docker_clear_cache()
 616
 617         runner = mock.MagicMock()
 618         runner.ignore_docker_for_reuse = False
 619         runner.intermediate_output_ttl = 0
 620         runner.secret_store = cwltool.secrets.SecretStore()
 621
 622         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 623         runner.api.collections().get().execute.return_value = {
 624             "portable_data_hash": "99999999999999999999999999999994+99",
 625             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 626
 627         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 628
 629         tool = cmap({
 630             "inputs": [
 631                 {"id": "p1",
 632                  "type": "Directory"}
 633             ],
 634             "outputs": [],
 635             "baseCommand": "ls",
 636             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 637             "id": "#",
 638             "class": "CommandLineTool"
 639         })
 640
 641         loadingContext, runtimeContext = self.helper(runner)
 642         runtimeContext.name = "test_run_mounts"
 643
 644         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 645         arvtool.formatgraph = None
 646         job_order = {
 647             "p1": {
 648                 "class": "Directory",
 649                 "location": "keep:99999999999999999999999999999994+44",
 650                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 651                 "listing": [
 652                     {
 653                         "class": "File",
 654                         "location": "keep:99999999999999999999999999999994+44/file1",
 655                     },
 656                     {
 657                         "class": "File",
 658                         "location": "keep:99999999999999999999999999999994+44/file2",
 659                     }
 660                 ]
 661             }
 662         }
 663         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 664             j.run(runtimeContext)
 665             runner.api.container_requests().create.assert_called_with(
 666                 body=JsonDiffMatcher({
 667                     'environment': {
 668                         'HOME': '/var/spool/cwl',
 669                         'TMPDIR': '/tmp'
 670                     },
 671                     'name': 'test_run_mounts',
 672                     'runtime_constraints': {
 673                         'vcpus': 1,
 674                         'ram': 1073741824
 675                     },
 676                     'use_existing': True,
 677                     'priority': 500,
 678                     'mounts': {
 679                         "/keep/99999999999999999999999999999994+44": {
 680                             "kind": "collection",
 681                             "portable_data_hash": "99999999999999999999999999999994+44",
 682                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 683                         },
 684                         '/tmp': {'kind': 'tmp',
 685                                  "capacity": 1073741824 },
 686                         '/var/spool/cwl': {'kind': 'tmp',
 687                                            "capacity": 1073741824 }
 688                     },
 689                     'state': 'Committed',
 690                     'output_name': 'Output for step test_run_mounts',
 691                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 692                     'output_path': '/var/spool/cwl',
 693                     'output_ttl': 0,
 694                     'container_image': '99999999999999999999999999999994+99',
 695                     'command': ['ls', '/var/spool/cwl'],
 696                     'cwd': '/var/spool/cwl',
 697                     'scheduling_parameters': {},
 698                     'properties': {},
 699                     'secret_mounts': {}
 700                 }))
 701
 702     # The test passes no builder.resources
 703     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 704     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 705     def test_secrets(self, keepdocker):
 706         arv_docker_clear_cache()
 707
 708         runner = mock.MagicMock()
 709         runner.ignore_docker_for_reuse = False
 710         runner.intermediate_output_ttl = 0
 711         runner.secret_store = cwltool.secrets.SecretStore()
 712
 713         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 714         runner.api.collections().get().execute.return_value = {
 715             "portable_data_hash": "99999999999999999999999999999993+99"}
 716
 717         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 718
 719         tool = cmap({"arguments": ["md5sum", "example.conf"],
 720                      "class": "CommandLineTool",
 721                      "hints": [
 722                          {
 723                              "class": "http://commonwl.org/cwltool#Secrets",
 724                              "secrets": [
 725                                  "#secret_job.cwl/pw"
 726                              ]
 727                          }
 728                      ],
 729                      "id": "#secret_job.cwl",
 730                      "inputs": [
 731                          {
 732                              "id": "#secret_job.cwl/pw",
 733                              "type": "string"
 734                          }
 735                      ],
 736                      "outputs": [
 737                      ],
 738                      "requirements": [
 739                          {
 740                              "class": "InitialWorkDirRequirement",
 741                              "listing": [
 742                                  {
 743                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 744                                      "entryname": "example.conf"
 745                                  }
 746                              ]
 747                          }
 748                      ]})
 749
 750         loadingContext, runtimeContext = self.helper(runner)
 751         runtimeContext.name = "test_secrets"
 752
 753         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 754         arvtool.formatgraph = None
 755
 756         job_order = {"pw": "blorp"}
 757         runner.secret_store.store(["pw"], job_order)
 758
 759         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 760             j.run(runtimeContext)
 761             runner.api.container_requests().create.assert_called_with(
 762                 body=JsonDiffMatcher({
 763                     'environment': {
 764                         'HOME': '/var/spool/cwl',
 765                         'TMPDIR': '/tmp'
 766                     },
 767                     'name': 'test_secrets',
 768                     'runtime_constraints': {
 769                         'vcpus': 1,
 770                         'ram': 1073741824
 771                     },
 772                     'use_existing': True,
 773                     'priority': 500,
 774                     'mounts': {
 775                         '/tmp': {'kind': 'tmp',
 776                                  "capacity": 1073741824
 777                              },
 778                         '/var/spool/cwl': {'kind': 'tmp',
 779                                            "capacity": 1073741824 }
 780                     },
 781                     'state': 'Committed',
 782                     'output_name': 'Output for step test_secrets',
 783                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 784                     'output_path': '/var/spool/cwl',
 785                     'output_ttl': 0,
 786                     'container_image': '99999999999999999999999999999993+99',
 787                     'command': ['md5sum', 'example.conf'],
 788                     'cwd': '/var/spool/cwl',
 789                     'scheduling_parameters': {},
 790                     'properties': {},
 791                     "secret_mounts": {
 792                         "/var/spool/cwl/example.conf": {
 793                             "content": "username: user\npassword: blorp\n",
 794                             "kind": "text"
 795                         }
 796                     }
 797                 }))
 798
 799     # The test passes no builder.resources
 800     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 801     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 802     def test_timelimit(self, keepdocker):
 803         arv_docker_clear_cache()
 804
 805         runner = mock.MagicMock()
 806         runner.ignore_docker_for_reuse = False
 807         runner.intermediate_output_ttl = 0
 808         runner.secret_store = cwltool.secrets.SecretStore()
 809
 810         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 811         runner.api.collections().get().execute.return_value = {
 812             "portable_data_hash": "99999999999999999999999999999993+99"}
 813
 814         tool = cmap({
 815             "inputs": [],
 816             "outputs": [],
 817             "baseCommand": "ls",
 818             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 819             "id": "#",
 820             "class": "CommandLineTool",
 821             "hints": [
 822                 {
 823                     "class": "ToolTimeLimit",
 824                     "timelimit": 42
 825                 }
 826             ]
 827         })
 828
 829         loadingContext, runtimeContext = self.helper(runner)
 830         runtimeContext.name = "test_timelimit"
 831
 832         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
 833         arvtool.formatgraph = None
 834
 835         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 836             j.run(runtimeContext)
 837
 838         _, kwargs = runner.api.container_requests().create.call_args
 839         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 840
 841
 842 class TestWorkflow(unittest.TestCase):
 843     def setUp(self):
 844         cwltool.process._names = set()
 845
 846     def helper(self, runner, enable_reuse=True):
 847         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 848
 849         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 850                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 851
 852         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
 853         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
 854         document_loader.fetch_text = document_loader.fetcher.fetch_text
 855         document_loader.check_exists = document_loader.fetcher.check_exists
 856
 857         loadingContext = arvados_cwl.context.ArvLoadingContext(
 858             {"avsc_names": avsc_names,
 859              "basedir": "",
 860              "make_fs_access": make_fs_access,
 861              "loader": document_loader,
 862              "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
 863              "construct_tool_object": runner.arv_make_tool})
 864         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
 865             {"work_api": "containers",
 866              "basedir": "",
 867              "name": "test_run_wf_"+str(enable_reuse),
 868              "make_fs_access": make_fs_access,
 869              "tmpdir": "/tmp",
 870              "enable_reuse": enable_reuse,
 871              "priority": 500})
 872
 873         return loadingContext, runtimeContext
 874
 875     # The test passes no builder.resources
 876     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 877     @mock.patch("arvados.collection.CollectionReader")
 878     @mock.patch("arvados.collection.Collection")
 879     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 880     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
 881         arv_docker_clear_cache()
 882         arvados_cwl.add_arv_hints()
 883
 884         api = mock.MagicMock()
 885         api._rootDesc = get_rootDesc()
 886
 887         runner = arvados_cwl.executor.ArvCwlExecutor(api)
 888         self.assertEqual(runner.work_api, 'containers')
 889
 890         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
 891         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
 892         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 893                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
 894
 895         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 896         runner.ignore_docker_for_reuse = False
 897         runner.num_retries = 0
 898         runner.secret_store = cwltool.secrets.SecretStore()
 899
 900         loadingContext, runtimeContext = self.helper(runner)
 901         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 902
 903         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
 904         metadata["cwlVersion"] = tool["cwlVersion"]
 905
 906         mockc = mock.MagicMock()
 907         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
 908         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 909
 910         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
 911         arvtool.formatgraph = None
 912         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 913
 914         next(it).run(runtimeContext)
 915         next(it).run(runtimeContext)
 916
 917         with open("tests/wf/scatter2_subwf.cwl") as f:
 918             subwf = StripYAMLComments(f.read()).rstrip()
 919
 920         runner.api.container_requests().create.assert_called_with(
 921             body=JsonDiffMatcher({
 922                 "command": [
 923                     "cwltool",
 924                     "--no-container",
 925                     "--move-outputs",
 926                     "--preserve-entire-environment",
 927                     "workflow.cwl#main",
 928                     "cwl.input.yml"
 929                 ],
 930                 "container_image": "99999999999999999999999999999993+99",
 931                 "cwd": "/var/spool/cwl",
 932                 "environment": {
 933                     "HOME": "/var/spool/cwl",
 934                     "TMPDIR": "/tmp"
 935                 },
 936                 "mounts": {
 937                     "/keep/99999999999999999999999999999999+118": {
 938                         "kind": "collection",
 939                         "portable_data_hash": "99999999999999999999999999999999+118"
 940                     },
 941                     "/tmp": {
 942                         "capacity": 1073741824,
 943                         "kind": "tmp"
 944                     },
 945                     "/var/spool/cwl": {
 946                         "capacity": 1073741824,
 947                         "kind": "tmp"
 948                     },
 949                     "/var/spool/cwl/cwl.input.yml": {
 950                         "kind": "collection",
 951                         "path": "cwl.input.yml",
 952                         "portable_data_hash": "99999999999999999999999999999996+99"
 953                     },
 954                     "/var/spool/cwl/workflow.cwl": {
 955                         "kind": "collection",
 956                         "path": "workflow.cwl",
 957                         "portable_data_hash": "99999999999999999999999999999996+99"
 958                     },
 959                     "stdout": {
 960                         "kind": "file",
 961                         "path": "/var/spool/cwl/cwl.output.json"
 962                     }
 963                 },
 964                 "name": "scatterstep",
 965                 "output_name": "Output for step scatterstep",
 966                 "output_path": "/var/spool/cwl",
 967                 "output_ttl": 0,
 968                 "priority": 500,
 969                 "properties": {},
 970                 "runtime_constraints": {
 971                     "ram": 1073741824,
 972                     "vcpus": 1
 973                 },
 974                 "scheduling_parameters": {},
 975                 "secret_mounts": {},
 976                 "state": "Committed",
 977                 "use_existing": True
 978             }))
 979         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
 980         mockc.open().__enter__().write.assert_has_calls([mock.call(
 981 '''{
 982   "fileblub": {
 983     "basename": "token.txt",
 984     "class": "File",
 985     "location": "/keep/99999999999999999999999999999999+118/token.txt",
 986     "size": 0
 987   },
 988   "sleeptime": 5
 989 }''')])
 990
 991     # The test passes no builder.resources
 992     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 993     @mock.patch("arvados.collection.CollectionReader")
 994     @mock.patch("arvados.collection.Collection")
 995     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
 996     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
 997         arv_docker_clear_cache()
 998         arvados_cwl.add_arv_hints()
 999
1000         api = mock.MagicMock()
1001         api._rootDesc = get_rootDesc()
1002
1003         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1004         self.assertEqual(runner.work_api, 'containers')
1005
1006         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1007         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1008                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1009         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1010                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1011
1012         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1013         runner.ignore_docker_for_reuse = False
1014         runner.num_retries = 0
1015         runner.secret_store = cwltool.secrets.SecretStore()
1016
1017         loadingContext, runtimeContext = self.helper(runner)
1018         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1019         loadingContext.do_update = True
1020         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1021
1022         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1023
1024         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1025         arvtool.formatgraph = None
1026         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1027
1028         next(it).run(runtimeContext)
1029         next(it).run(runtimeContext)
1030
1031         with open("tests/wf/echo-subwf.cwl") as f:
1032             subwf = StripYAMLComments(f.read())
1033
1034         runner.api.container_requests().create.assert_called_with(
1035             body=JsonDiffMatcher({
1036                 'output_ttl': 0,
1037                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1038                 'scheduling_parameters': {},
1039                 'name': u'echo-subwf',
1040                 'secret_mounts': {},
1041                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1042                 'properties': {},
1043                 'priority': 500,
1044                 'mounts': {
1045                     '/var/spool/cwl/cwl.input.yml': {
1046                         'portable_data_hash': '99999999999999999999999999999996+99',
1047                         'kind': 'collection',
1048                         'path': 'cwl.input.yml'
1049                     },
1050                     '/var/spool/cwl/workflow.cwl': {
1051                         'portable_data_hash': '99999999999999999999999999999996+99',
1052                         'kind': 'collection',
1053                         'path': 'workflow.cwl'
1054                     },
1055                     'stdout': {
1056                         'path': '/var/spool/cwl/cwl.output.json',
1057                         'kind': 'file'
1058                     },
1059                     '/tmp': {
1060                         'kind': 'tmp',
1061                         'capacity': 1073741824
1062                     }, '/var/spool/cwl': {
1063                         'kind': 'tmp',
1064                         'capacity': 3221225472
1065                     }
1066                 },
1067                 'state': 'Committed',
1068                 'output_path': '/var/spool/cwl',
1069                 'container_image': '99999999999999999999999999999993+99',
1070                 'command': [
1071                     u'cwltool',
1072                     u'--no-container',
1073                     u'--move-outputs',
1074                     u'--preserve-entire-environment',
1075                     u'workflow.cwl#main',
1076                     u'cwl.input.yml'
1077                 ],
1078                 'use_existing': True,
1079                 'output_name': u'Output for step echo-subwf',
1080                 'cwd': '/var/spool/cwl'
1081             }))
1082
1083     def test_default_work_api(self):
1084         arvados_cwl.add_arv_hints()
1085
1086         api = mock.MagicMock()
1087         api._rootDesc = copy.deepcopy(get_rootDesc())
1088         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1089         self.assertEqual(runner.work_api, 'containers')