sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 import cwltool.load_tool
  22 from cwltool.update import INTERNAL_VERSION
  23 from schema_salad.ref_resolver import Loader
  24 from schema_salad.sourceline import cmap
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         arv_docker_clear_cache()
  64
  65     def helper(self, runner, enable_reuse=True):
  66         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  67
  68         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  69                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  70         fs_access = mock.MagicMock()
  71         fs_access.exists.return_value = True
  72
  73         loadingContext = arvados_cwl.context.ArvLoadingContext(
  74             {"avsc_names": avsc_names,
  75              "basedir": "",
  76              "make_fs_access": make_fs_access,
  77              "construct_tool_object": runner.arv_make_tool,
  78              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
  79              })
  80         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  81             {"work_api": "containers",
  82              "basedir": "",
  83              "name": "test_run_"+str(enable_reuse),
  84              "make_fs_access": make_fs_access,
  85              "tmpdir": "/tmp",
  86              "enable_reuse": enable_reuse,
  87              "priority": 500,
  88              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  89             })
  90
  91         if isinstance(runner, mock.MagicMock):
  92             def make_tool(toolpath_object, loadingContext):
  93                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
  94             runner.arv_make_tool.side_effect = make_tool
  95
  96         return loadingContext, runtimeContext
  97
  98     # Helper function to set up the ArvCwlExecutor to use the containers api
  99     # and test that the RuntimeStatusLoggingHandler is set up correctly
 100     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 101         api = mock.MagicMock()
 102         api._rootDesc = copy.deepcopy(get_rootDesc())
 103
 104         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 105         # adds the logging handler that will call runtime_status_update() mock
 106         self.assertFalse(gcc_mock.called)
 107         runner = arvados_cwl.ArvCwlExecutor(api)
 108         self.assertEqual(runner.work_api, 'containers')
 109         root_logger = logging.getLogger('')
 110         handlerClasses = [h.__class__ for h in root_logger.handlers]
 111         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 112         return runner
 113
 114     # The test passes no builder.resources
 115     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 116     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 117     def test_run(self, keepdocker):
 118         for enable_reuse in (True, False):
 119             arv_docker_clear_cache()
 120
 121             runner = mock.MagicMock()
 122             runner.ignore_docker_for_reuse = False
 123             runner.intermediate_output_ttl = 0
 124             runner.secret_store = cwltool.secrets.SecretStore()
 125             runner.api._rootDesc = {"revision": "20210628"}
 126
 127             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 128             runner.api.collections().get().execute.return_value = {
 129                 "portable_data_hash": "99999999999999999999999999999993+99"}
 130
 131             tool = cmap({
 132                 "inputs": [],
 133                 "outputs": [],
 134                 "baseCommand": "ls",
 135                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 136                 "id": "",
 137                 "class": "CommandLineTool",
 138                 "cwlVersion": "v1.2"
 139             })
 140
 141             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 142
 143             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 144             arvtool.formatgraph = None
 145
 146             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 147                 j.run(runtimeContext)
 148                 runner.api.container_requests().create.assert_called_with(
 149                     body=JsonDiffMatcher({
 150                         'environment': {
 151                             'HOME': '/var/spool/cwl',
 152                             'TMPDIR': '/tmp'
 153                         },
 154                         'name': 'test_run_'+str(enable_reuse),
 155                         'runtime_constraints': {
 156                             'vcpus': 1,
 157                             'ram': 268435456
 158                         },
 159                         'use_existing': enable_reuse,
 160                         'priority': 500,
 161                         'mounts': {
 162                             '/tmp': {'kind': 'tmp',
 163                                      "capacity": 1073741824
 164                                  },
 165                             '/var/spool/cwl': {'kind': 'tmp',
 166                                                "capacity": 1073741824 }
 167                         },
 168                         'state': 'Committed',
 169                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 170                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 171                         'output_path': '/var/spool/cwl',
 172                         'output_ttl': 0,
 173                         'container_image': '99999999999999999999999999999993+99',
 174                         'command': ['ls', '/var/spool/cwl'],
 175                         'cwd': '/var/spool/cwl',
 176                         'scheduling_parameters': {},
 177                         'properties': {},
 178                         'secret_mounts': {},
 179                         'output_storage_classes': ["default"]
 180                     }))
 181
 182     # The test passes some fields in builder.resources
 183     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 184     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 185     def test_resource_requirements(self, keepdocker):
 186         arvados_cwl.add_arv_hints()
 187         runner = mock.MagicMock()
 188         runner.ignore_docker_for_reuse = False
 189         runner.intermediate_output_ttl = 3600
 190         runner.secret_store = cwltool.secrets.SecretStore()
 191         runner.api._rootDesc = {"revision": "20210628"}
 192
 193         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 194         runner.api.collections().get().execute.return_value = {
 195             "portable_data_hash": "99999999999999999999999999999993+99"}
 196
 197         tool = cmap({
 198             "inputs": [],
 199             "outputs": [],
 200             "hints": [{
 201                 "class": "ResourceRequirement",
 202                 "coresMin": 3,
 203                 "ramMin": 3000,
 204                 "tmpdirMin": 4000,
 205                 "outdirMin": 5000
 206             }, {
 207                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 208                 "keep_cache": 512
 209             }, {
 210                 "class": "http://arvados.org/cwl#APIRequirement",
 211             }, {
 212                 "class": "http://arvados.org/cwl#PartitionRequirement",
 213                 "partition": "blurb"
 214             }, {
 215                 "class": "http://arvados.org/cwl#IntermediateOutput",
 216                 "outputTTL": 7200
 217             }, {
 218                 "class": "WorkReuse",
 219                 "enableReuse": False
 220             }],
 221             "baseCommand": "ls",
 222             "id": "",
 223             "class": "CommandLineTool",
 224             "cwlVersion": "v1.2"
 225         })
 226
 227         loadingContext, runtimeContext = self.helper(runner)
 228         runtimeContext.name = "test_resource_requirements"
 229
 230         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 231         arvtool.formatgraph = None
 232         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 233             j.run(runtimeContext)
 234
 235         call_args, call_kwargs = runner.api.container_requests().create.call_args
 236
 237         call_body_expected = {
 238             'environment': {
 239                 'HOME': '/var/spool/cwl',
 240                 'TMPDIR': '/tmp'
 241             },
 242             'name': 'test_resource_requirements',
 243             'runtime_constraints': {
 244                 'vcpus': 3,
 245                 'ram': 3145728000,
 246                 'keep_cache_ram': 536870912,
 247                 'API': True
 248             },
 249             'use_existing': False,
 250             'priority': 500,
 251             'mounts': {
 252                 '/tmp': {'kind': 'tmp',
 253                          "capacity": 4194304000 },
 254                 '/var/spool/cwl': {'kind': 'tmp',
 255                                    "capacity": 5242880000 }
 256             },
 257             'state': 'Committed',
 258             'output_name': 'Output for step test_resource_requirements',
 259             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 260             'output_path': '/var/spool/cwl',
 261             'output_ttl': 7200,
 262             'container_image': '99999999999999999999999999999993+99',
 263             'command': ['ls'],
 264             'cwd': '/var/spool/cwl',
 265             'scheduling_parameters': {
 266                 'partitions': ['blurb']
 267             },
 268             'properties': {},
 269             'secret_mounts': {},
 270             'output_storage_classes': ["default"]
 271         }
 272
 273         call_body = call_kwargs.get('body', None)
 274         self.assertNotEqual(None, call_body)
 275         for key in call_body:
 276             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 277
 278
 279     # The test passes some fields in builder.resources
 280     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 281     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 282     @mock.patch("arvados.collection.Collection")
 283     def test_initial_work_dir(self, collection_mock, keepdocker):
 284         runner = mock.MagicMock()
 285         runner.ignore_docker_for_reuse = False
 286         runner.intermediate_output_ttl = 0
 287         runner.secret_store = cwltool.secrets.SecretStore()
 288         runner.api._rootDesc = {"revision": "20210628"}
 289
 290         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 291         runner.api.collections().get().execute.return_value = {
 292             "portable_data_hash": "99999999999999999999999999999993+99"}
 293
 294         sourcemock = mock.MagicMock()
 295         def get_collection_mock(p):
 296             if "/" in p:
 297                 return (sourcemock, p.split("/", 1)[1])
 298             else:
 299                 return (sourcemock, "")
 300         runner.fs_access.get_collection.side_effect = get_collection_mock
 301
 302         vwdmock = mock.MagicMock()
 303         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 304
 305         tool = cmap({
 306             "inputs": [],
 307             "outputs": [],
 308             "hints": [{
 309                 "class": "InitialWorkDirRequirement",
 310                 "listing": [{
 311                     "class": "File",
 312                     "basename": "foo",
 313                     "location": "keep:99999999999999999999999999999995+99/bar"
 314                 },
 315                 {
 316                     "class": "Directory",
 317                     "basename": "foo2",
 318                     "location": "keep:99999999999999999999999999999995+99"
 319                 },
 320                 {
 321                     "class": "File",
 322                     "basename": "filename",
 323                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 324                 },
 325                 {
 326                     "class": "Directory",
 327                     "basename": "subdir",
 328                     "location": "keep:99999999999999999999999999999995+99/subdir"
 329                 }                        ]
 330             }],
 331             "baseCommand": "ls",
 332             "class": "CommandLineTool",
 333             "cwlVersion": "v1.2",
 334             "id": ""
 335         })
 336
 337         loadingContext, runtimeContext = self.helper(runner)
 338         runtimeContext.name = "test_initial_work_dir"
 339
 340         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 341
 342         arvtool.formatgraph = None
 343         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 344             j.run(runtimeContext)
 345
 346         call_args, call_kwargs = runner.api.container_requests().create.call_args
 347
 348         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 349         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 350         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 351         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 352
 353         call_body_expected = {
 354             'environment': {
 355                 'HOME': '/var/spool/cwl',
 356                 'TMPDIR': '/tmp'
 357             },
 358             'name': 'test_initial_work_dir',
 359             'runtime_constraints': {
 360                 'vcpus': 1,
 361                 'ram': 268435456
 362             },
 363             'use_existing': True,
 364             'priority': 500,
 365             'mounts': {
 366                 '/tmp': {'kind': 'tmp',
 367                          "capacity": 1073741824 },
 368                 '/var/spool/cwl': {'kind': 'tmp',
 369                                    "capacity": 1073741824 },
 370                 '/var/spool/cwl/foo': {
 371                     'kind': 'collection',
 372                     'path': 'foo',
 373                     'portable_data_hash': '99999999999999999999999999999996+99'
 374                 },
 375                 '/var/spool/cwl/foo2': {
 376                     'kind': 'collection',
 377                     'path': 'foo2',
 378                     'portable_data_hash': '99999999999999999999999999999996+99'
 379                 },
 380                 '/var/spool/cwl/filename': {
 381                     'kind': 'collection',
 382                     'path': 'filename',
 383                     'portable_data_hash': '99999999999999999999999999999996+99'
 384                 },
 385                 '/var/spool/cwl/subdir': {
 386                     'kind': 'collection',
 387                     'path': 'subdir',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 }
 390             },
 391             'state': 'Committed',
 392             'output_name': 'Output for step test_initial_work_dir',
 393             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 394             'output_path': '/var/spool/cwl',
 395             'output_ttl': 0,
 396             'container_image': '99999999999999999999999999999993+99',
 397             'command': ['ls'],
 398             'cwd': '/var/spool/cwl',
 399             'scheduling_parameters': {
 400             },
 401             'properties': {},
 402             'secret_mounts': {},
 403             'output_storage_classes': ["default"]
 404         }
 405
 406         call_body = call_kwargs.get('body', None)
 407         self.assertNotEqual(None, call_body)
 408         for key in call_body:
 409             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 410
 411
 412     # Test redirecting stdin/stdout/stderr
 413     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 414     def test_redirects(self, keepdocker):
 415         runner = mock.MagicMock()
 416         runner.ignore_docker_for_reuse = False
 417         runner.intermediate_output_ttl = 0
 418         runner.secret_store = cwltool.secrets.SecretStore()
 419         runner.api._rootDesc = {"revision": "20210628"}
 420
 421         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 422         runner.api.collections().get().execute.return_value = {
 423             "portable_data_hash": "99999999999999999999999999999993+99"}
 424
 425         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 426
 427         tool = cmap({
 428             "inputs": [],
 429             "outputs": [],
 430             "baseCommand": "ls",
 431             "stdout": "stdout.txt",
 432             "stderr": "stderr.txt",
 433             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 434             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 435             "id": "",
 436             "class": "CommandLineTool",
 437             "cwlVersion": "v1.2"
 438         })
 439
 440         loadingContext, runtimeContext = self.helper(runner)
 441         runtimeContext.name = "test_run_redirect"
 442
 443         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 444         arvtool.formatgraph = None
 445         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 446             j.run(runtimeContext)
 447             runner.api.container_requests().create.assert_called_with(
 448                 body=JsonDiffMatcher({
 449                     'environment': {
 450                         'HOME': '/var/spool/cwl',
 451                         'TMPDIR': '/tmp'
 452                     },
 453                     'name': 'test_run_redirect',
 454                     'runtime_constraints': {
 455                         'vcpus': 1,
 456                         'ram': 268435456
 457                     },
 458                     'use_existing': True,
 459                     'priority': 500,
 460                     'mounts': {
 461                         '/tmp': {'kind': 'tmp',
 462                                  "capacity": 1073741824 },
 463                         '/var/spool/cwl': {'kind': 'tmp',
 464                                            "capacity": 1073741824 },
 465                         "stderr": {
 466                             "kind": "file",
 467                             "path": "/var/spool/cwl/stderr.txt"
 468                         },
 469                         "stdin": {
 470                             "kind": "collection",
 471                             "path": "file.txt",
 472                             "portable_data_hash": "99999999999999999999999999999996+99"
 473                         },
 474                         "stdout": {
 475                             "kind": "file",
 476                             "path": "/var/spool/cwl/stdout.txt"
 477                         },
 478                     },
 479                     'state': 'Committed',
 480                     "output_name": "Output for step test_run_redirect",
 481                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 482                     'output_path': '/var/spool/cwl',
 483                     'output_ttl': 0,
 484                     'container_image': '99999999999999999999999999999993+99',
 485                     'command': ['ls', '/var/spool/cwl'],
 486                     'cwd': '/var/spool/cwl',
 487                     'scheduling_parameters': {},
 488                     'properties': {},
 489                     'secret_mounts': {},
 490                     'output_storage_classes': ["default"]
 491                 }))
 492
 493     @mock.patch("arvados.collection.Collection")
 494     def test_done(self, col):
 495         api = mock.MagicMock()
 496
 497         runner = mock.MagicMock()
 498         runner.api = api
 499         runner.num_retries = 0
 500         runner.ignore_docker_for_reuse = False
 501         runner.intermediate_output_ttl = 0
 502         runner.secret_store = cwltool.secrets.SecretStore()
 503
 504         runner.api.containers().get().execute.return_value = {"state":"Complete",
 505                                                               "output": "abc+123",
 506                                                               "exit_code": 0}
 507
 508         col().open.return_value = []
 509
 510         loadingContext, runtimeContext = self.helper(runner)
 511
 512         arvjob = arvados_cwl.ArvadosContainer(runner,
 513                                               runtimeContext,
 514                                               mock.MagicMock(),
 515                                               {},
 516                                               None,
 517                                               [],
 518                                               [],
 519                                               "testjob")
 520         arvjob.output_callback = mock.MagicMock()
 521         arvjob.collect_outputs = mock.MagicMock()
 522         arvjob.successCodes = [0]
 523         arvjob.outdir = "/var/spool/cwl"
 524         arvjob.output_ttl = 3600
 525
 526         arvjob.collect_outputs.return_value = {"out": "stuff"}
 527
 528         arvjob.done({
 529             "state": "Final",
 530             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 531             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 532             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 533             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 534             "modified_at": "2017-05-26T12:01:22Z"
 535         })
 536
 537         self.assertFalse(api.collections().create.called)
 538         self.assertFalse(runner.runtime_status_error.called)
 539
 540         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 541         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 542         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 543
 544     # Test to make sure we dont call runtime_status_update if we already did
 545     # some where higher up in the call stack
 546     @mock.patch("arvados_cwl.util.get_current_container")
 547     def test_recursive_runtime_status_update(self, gcc_mock):
 548         self.setup_and_test_container_executor_and_logging(gcc_mock)
 549         root_logger = logging.getLogger('')
 550
 551         # get_current_container is invoked when we call runtime_status_update
 552         # so try and log again!
 553         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 554         try:
 555             root_logger.error("First Error")
 556         except RuntimeError:
 557             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 558
 559
 560     # Test to make sure that an exception raised from
 561     # get_current_container doesn't cause the logger to raise an
 562     # exception
 563     @mock.patch("arvados_cwl.util.get_current_container")
 564     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 565         self.setup_and_test_container_executor_and_logging(gcc_mock)
 566         root_logger = logging.getLogger('')
 567
 568         # get_current_container is invoked when we call
 569         # runtime_status_update, it is going to also raise an
 570         # exception.
 571         gcc_mock.side_effect = Exception("Second Error")
 572         try:
 573             root_logger.error("First Error")
 574         except Exception:
 575             self.fail("Exception in logger should not propagate")
 576         self.assertTrue(gcc_mock.called)
 577
 578     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 579     @mock.patch("arvados_cwl.util.get_current_container")
 580     @mock.patch("arvados.collection.CollectionReader")
 581     @mock.patch("arvados.collection.Collection")
 582     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 583         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 584
 585         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 586         self.assertTrue(gcc_mock.called)
 587
 588         runner.num_retries = 0
 589         runner.ignore_docker_for_reuse = False
 590         runner.intermediate_output_ttl = 0
 591         runner.secret_store = cwltool.secrets.SecretStore()
 592         runner.label = mock.MagicMock()
 593         runner.label.return_value = '[container testjob]'
 594
 595         runner.api.containers().get().execute.return_value = {
 596             "state":"Complete",
 597             "output": "abc+123",
 598             "exit_code": 1,
 599             "log": "def+234"
 600         }
 601
 602         col().open.return_value = []
 603
 604         loadingContext, runtimeContext = self.helper(runner)
 605
 606         arvjob = arvados_cwl.ArvadosContainer(runner,
 607                                               runtimeContext,
 608                                               mock.MagicMock(),
 609                                               {},
 610                                               None,
 611                                               [],
 612                                               [],
 613                                               "testjob")
 614         arvjob.output_callback = mock.MagicMock()
 615         arvjob.collect_outputs = mock.MagicMock()
 616         arvjob.successCodes = [0]
 617         arvjob.outdir = "/var/spool/cwl"
 618         arvjob.output_ttl = 3600
 619         arvjob.collect_outputs.return_value = {"out": "stuff"}
 620
 621         arvjob.done({
 622             "state": "Final",
 623             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 624             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 625             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 626             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 627             "modified_at": "2017-05-26T12:01:22Z"
 628         })
 629
 630         rts_mock.assert_called_with(
 631             'error',
 632             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 633             '  ** log is empty **'
 634         )
 635         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 636
 637     # The test passes no builder.resources
 638     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 639     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 640     def test_mounts(self, keepdocker):
 641         runner = mock.MagicMock()
 642         runner.ignore_docker_for_reuse = False
 643         runner.intermediate_output_ttl = 0
 644         runner.secret_store = cwltool.secrets.SecretStore()
 645         runner.api._rootDesc = {"revision": "20210628"}
 646
 647         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 648         runner.api.collections().get().execute.return_value = {
 649             "portable_data_hash": "99999999999999999999999999999994+99",
 650             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 651
 652         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 653
 654         tool = cmap({
 655             "inputs": [
 656                 {"id": "p1",
 657                  "type": "Directory"}
 658             ],
 659             "outputs": [],
 660             "baseCommand": "ls",
 661             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 662             "id": "",
 663             "class": "CommandLineTool",
 664             "cwlVersion": "v1.2"
 665         })
 666
 667         loadingContext, runtimeContext = self.helper(runner)
 668         runtimeContext.name = "test_run_mounts"
 669
 670         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 671         arvtool.formatgraph = None
 672         job_order = {
 673             "p1": {
 674                 "class": "Directory",
 675                 "location": "keep:99999999999999999999999999999994+44",
 676                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 677                 "listing": [
 678                     {
 679                         "class": "File",
 680                         "location": "keep:99999999999999999999999999999994+44/file1",
 681                     },
 682                     {
 683                         "class": "File",
 684                         "location": "keep:99999999999999999999999999999994+44/file2",
 685                     }
 686                 ]
 687             }
 688         }
 689         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 690             j.run(runtimeContext)
 691             runner.api.container_requests().create.assert_called_with(
 692                 body=JsonDiffMatcher({
 693                     'environment': {
 694                         'HOME': '/var/spool/cwl',
 695                         'TMPDIR': '/tmp'
 696                     },
 697                     'name': 'test_run_mounts',
 698                     'runtime_constraints': {
 699                         'vcpus': 1,
 700                         'ram': 268435456
 701                     },
 702                     'use_existing': True,
 703                     'priority': 500,
 704                     'mounts': {
 705                         "/keep/99999999999999999999999999999994+44": {
 706                             "kind": "collection",
 707                             "portable_data_hash": "99999999999999999999999999999994+44",
 708                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 709                         },
 710                         '/tmp': {'kind': 'tmp',
 711                                  "capacity": 1073741824 },
 712                         '/var/spool/cwl': {'kind': 'tmp',
 713                                            "capacity": 1073741824 }
 714                     },
 715                     'state': 'Committed',
 716                     'output_name': 'Output for step test_run_mounts',
 717                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 718                     'output_path': '/var/spool/cwl',
 719                     'output_ttl': 0,
 720                     'container_image': '99999999999999999999999999999994+99',
 721                     'command': ['ls', '/var/spool/cwl'],
 722                     'cwd': '/var/spool/cwl',
 723                     'scheduling_parameters': {},
 724                     'properties': {},
 725                     'secret_mounts': {},
 726                     'output_storage_classes': ["default"]
 727                 }))
 728
 729     # The test passes no builder.resources
 730     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 731     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 732     def test_secrets(self, keepdocker):
 733         arvados_cwl.add_arv_hints()
 734         runner = mock.MagicMock()
 735         runner.ignore_docker_for_reuse = False
 736         runner.intermediate_output_ttl = 0
 737         runner.secret_store = cwltool.secrets.SecretStore()
 738         runner.api._rootDesc = {"revision": "20210628"}
 739
 740         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 741         runner.api.collections().get().execute.return_value = {
 742             "portable_data_hash": "99999999999999999999999999999993+99"}
 743
 744         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 745
 746         tool = cmap({"arguments": ["md5sum", "example.conf"],
 747                      "class": "CommandLineTool",
 748                      "cwlVersion": "v1.2",
 749                      "hints": [
 750                          {
 751                              "class": "http://commonwl.org/cwltool#Secrets",
 752                              "secrets": [
 753                                  "#secret_job.cwl/pw"
 754                              ]
 755                          }
 756                      ],
 757                      "id": "",
 758                      "inputs": [
 759                          {
 760                              "id": "#secret_job.cwl/pw",
 761                              "type": "string"
 762                          }
 763                      ],
 764                      "outputs": [
 765                      ],
 766                      "requirements": [
 767                          {
 768                              "class": "InitialWorkDirRequirement",
 769                              "listing": [
 770                                  {
 771                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 772                                      "entryname": "example.conf"
 773                                  }
 774                              ]
 775                          }
 776                      ]})
 777
 778         loadingContext, runtimeContext = self.helper(runner)
 779         runtimeContext.name = "test_secrets"
 780
 781         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 782         arvtool.formatgraph = None
 783
 784         job_order = {"pw": "blorp"}
 785         runner.secret_store.store(["pw"], job_order)
 786
 787         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 788             j.run(runtimeContext)
 789             runner.api.container_requests().create.assert_called_with(
 790                 body=JsonDiffMatcher({
 791                     'environment': {
 792                         'HOME': '/var/spool/cwl',
 793                         'TMPDIR': '/tmp'
 794                     },
 795                     'name': 'test_secrets',
 796                     'runtime_constraints': {
 797                         'vcpus': 1,
 798                         'ram': 268435456
 799                     },
 800                     'use_existing': True,
 801                     'priority': 500,
 802                     'mounts': {
 803                         '/tmp': {'kind': 'tmp',
 804                                  "capacity": 1073741824
 805                              },
 806                         '/var/spool/cwl': {'kind': 'tmp',
 807                                            "capacity": 1073741824 }
 808                     },
 809                     'state': 'Committed',
 810                     'output_name': 'Output for step test_secrets',
 811                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 812                     'output_path': '/var/spool/cwl',
 813                     'output_ttl': 0,
 814                     'container_image': '99999999999999999999999999999993+99',
 815                     'command': ['md5sum', 'example.conf'],
 816                     'cwd': '/var/spool/cwl',
 817                     'scheduling_parameters': {},
 818                     'properties': {},
 819                     "secret_mounts": {
 820                         "/var/spool/cwl/example.conf": {
 821                             "content": "username: user\npassword: blorp\n",
 822                             "kind": "text"
 823                         }
 824                     },
 825                     'output_storage_classes': ["default"]
 826                 }))
 827
 828     # The test passes no builder.resources
 829     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 830     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 831     def test_timelimit(self, keepdocker):
 832         runner = mock.MagicMock()
 833         runner.ignore_docker_for_reuse = False
 834         runner.intermediate_output_ttl = 0
 835         runner.secret_store = cwltool.secrets.SecretStore()
 836         runner.api._rootDesc = {"revision": "20210628"}
 837
 838         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 839         runner.api.collections().get().execute.return_value = {
 840             "portable_data_hash": "99999999999999999999999999999993+99"}
 841
 842         tool = cmap({
 843             "inputs": [],
 844             "outputs": [],
 845             "baseCommand": "ls",
 846             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 847             "id": "",
 848             "cwlVersion": "v1.2",
 849             "class": "CommandLineTool",
 850             "hints": [
 851                 {
 852                     "class": "ToolTimeLimit",
 853                     "timelimit": 42
 854                 }
 855             ]
 856         })
 857
 858         loadingContext, runtimeContext = self.helper(runner)
 859         runtimeContext.name = "test_timelimit"
 860
 861         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 862         arvtool.formatgraph = None
 863
 864         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 865             j.run(runtimeContext)
 866
 867         _, kwargs = runner.api.container_requests().create.call_args
 868         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 869
 870
 871     # The test passes no builder.resources
 872     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 873     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 874     def test_setting_storage_class(self, keepdocker):
 875         arv_docker_clear_cache()
 876
 877         runner = mock.MagicMock()
 878         runner.ignore_docker_for_reuse = False
 879         runner.intermediate_output_ttl = 0
 880         runner.secret_store = cwltool.secrets.SecretStore()
 881         runner.api._rootDesc = {"revision": "20210628"}
 882
 883         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 884         runner.api.collections().get().execute.return_value = {
 885             "portable_data_hash": "99999999999999999999999999999993+99"}
 886
 887         tool = cmap({
 888             "inputs": [],
 889             "outputs": [],
 890             "baseCommand": "ls",
 891             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 892             "id": "",
 893             "cwlVersion": "v1.2",
 894             "class": "CommandLineTool",
 895             "hints": [
 896                 {
 897                     "class": "http://arvados.org/cwl#OutputStorageClass",
 898                     "finalStorageClass": ["baz_sc", "qux_sc"],
 899                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 900                 }
 901             ]
 902         })
 903
 904         loadingContext, runtimeContext = self.helper(runner, True)
 905
 906         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 907         arvtool.formatgraph = None
 908
 909         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 910             j.run(runtimeContext)
 911             runner.api.container_requests().create.assert_called_with(
 912                 body=JsonDiffMatcher({
 913                     'environment': {
 914                         'HOME': '/var/spool/cwl',
 915                         'TMPDIR': '/tmp'
 916                     },
 917                     'name': 'test_run_True',
 918                     'runtime_constraints': {
 919                         'vcpus': 1,
 920                         'ram': 268435456
 921                     },
 922                     'use_existing': True,
 923                     'priority': 500,
 924                     'mounts': {
 925                         '/tmp': {'kind': 'tmp',
 926                                  "capacity": 1073741824
 927                              },
 928                         '/var/spool/cwl': {'kind': 'tmp',
 929                                            "capacity": 1073741824 }
 930                     },
 931                     'state': 'Committed',
 932                     'output_name': 'Output for step test_run_True',
 933                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 934                     'output_path': '/var/spool/cwl',
 935                     'output_ttl': 0,
 936                     'container_image': '99999999999999999999999999999993+99',
 937                     'command': ['ls', '/var/spool/cwl'],
 938                     'cwd': '/var/spool/cwl',
 939                     'scheduling_parameters': {},
 940                     'properties': {},
 941                     'secret_mounts': {},
 942                     'output_storage_classes': ["foo_sc", "bar_sc"]
 943                 }))
 944
 945
 946     # The test passes no builder.resources
 947     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 948     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 949     def test_setting_process_properties(self, keepdocker):
 950         arv_docker_clear_cache()
 951
 952         runner = mock.MagicMock()
 953         runner.ignore_docker_for_reuse = False
 954         runner.intermediate_output_ttl = 0
 955         runner.secret_store = cwltool.secrets.SecretStore()
 956         runner.api._rootDesc = {"revision": "20210628"}
 957
 958         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 959         runner.api.collections().get().execute.return_value = {
 960             "portable_data_hash": "99999999999999999999999999999993+99"}
 961
 962         tool = cmap({
 963             "inputs": [
 964                 {"id": "x", "type": "string"}],
 965             "outputs": [],
 966             "baseCommand": "ls",
 967             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 968             "id": "",
 969             "class": "CommandLineTool",
 970             "cwlVersion": "v1.2",
 971             "hints": [
 972             {
 973                 "class": "http://arvados.org/cwl#ProcessProperties",
 974                 "processProperties": [
 975                     {"propertyName": "foo",
 976                      "propertyValue": "bar"},
 977                     {"propertyName": "baz",
 978                      "propertyValue": "$(inputs.x)"},
 979                     {"propertyName": "quux",
 980                      "propertyValue": {
 981                          "q1": 1,
 982                          "q2": 2
 983                      }
 984                     }
 985                 ],
 986             }
 987         ]
 988         })
 989
 990         loadingContext, runtimeContext = self.helper(runner, True)
 991
 992         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 993         arvtool.formatgraph = None
 994
 995         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 996             j.run(runtimeContext)
 997             runner.api.container_requests().create.assert_called_with(
 998                 body=JsonDiffMatcher({
 999                     'environment': {
1000                         'HOME': '/var/spool/cwl',
1001                         'TMPDIR': '/tmp'
1002                     },
1003                     'name': 'test_run_True',
1004                     'runtime_constraints': {
1005                         'vcpus': 1,
1006                         'ram': 268435456
1007                     },
1008                     'use_existing': True,
1009                     'priority': 500,
1010                     'mounts': {
1011                         '/tmp': {'kind': 'tmp',
1012                                  "capacity": 1073741824
1013                              },
1014                         '/var/spool/cwl': {'kind': 'tmp',
1015                                            "capacity": 1073741824 }
1016                     },
1017                     'state': 'Committed',
1018                     'output_name': 'Output for step test_run_True',
1019                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1020                     'output_path': '/var/spool/cwl',
1021                     'output_ttl': 0,
1022                     'container_image': '99999999999999999999999999999993+99',
1023                     'command': ['ls', '/var/spool/cwl'],
1024                     'cwd': '/var/spool/cwl',
1025                     'scheduling_parameters': {},
1026                     'properties': {
1027                         "baz": "blorp",
1028                         "foo": "bar",
1029                         "quux": {
1030                             "q1": 1,
1031                             "q2": 2
1032                         }
1033                     },
1034                     'secret_mounts': {},
1035                     'output_storage_classes': ["default"]
1036                 }))
1037
1038
1039     # The test passes no builder.resources
1040     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1041     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1042     def test_cuda_requirement(self, keepdocker):
1043         arvados_cwl.add_arv_hints()
1044         arv_docker_clear_cache()
1045
1046         runner = mock.MagicMock()
1047         runner.ignore_docker_for_reuse = False
1048         runner.intermediate_output_ttl = 0
1049         runner.secret_store = cwltool.secrets.SecretStore()
1050         runner.api._rootDesc = {"revision": "20210628"}
1051
1052         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1053         runner.api.collections().get().execute.return_value = {
1054             "portable_data_hash": "99999999999999999999999999999993+99"}
1055
1056         test_cwl_req = [{
1057                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1058                 "cudaVersionMin": "11.0",
1059                 "cudaComputeCapability": "9.0",
1060             }, {
1061                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1062                 "cudaVersionMin": "11.0",
1063                 "cudaComputeCapability": "9.0",
1064                 "cudaDeviceCountMin": 2
1065             }, {
1066                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1067                 "cudaVersionMin": "11.0",
1068                 "cudaComputeCapability": ["4.0", "5.0"],
1069                 "cudaDeviceCountMin": 2
1070             }]
1071
1072         test_arv_req = [{
1073             'device_count': 1,
1074             'driver_version': "11.0",
1075             'hardware_capability': "9.0"
1076         }, {
1077             'device_count': 2,
1078             'driver_version': "11.0",
1079             'hardware_capability': "9.0"
1080         }, {
1081             'device_count': 2,
1082             'driver_version': "11.0",
1083             'hardware_capability': "4.0"
1084         }]
1085
1086         for test_case in range(0, len(test_cwl_req)):
1087
1088             tool = cmap({
1089                 "inputs": [],
1090                 "outputs": [],
1091                 "baseCommand": "nvidia-smi",
1092                 "arguments": [],
1093                 "id": "",
1094                 "cwlVersion": "v1.2",
1095                 "class": "CommandLineTool",
1096                 "requirements": [test_cwl_req[test_case]]
1097             })
1098
1099             loadingContext, runtimeContext = self.helper(runner, True)
1100
1101             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1102             arvtool.formatgraph = None
1103
1104             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1105                 j.run(runtimeContext)
1106                 runner.api.container_requests().create.assert_called_with(
1107                     body=JsonDiffMatcher({
1108                         'environment': {
1109                             'HOME': '/var/spool/cwl',
1110                             'TMPDIR': '/tmp'
1111                         },
1112                         'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1113                         'runtime_constraints': {
1114                             'vcpus': 1,
1115                             'ram': 268435456,
1116                             'cuda': test_arv_req[test_case]
1117                         },
1118                         'use_existing': True,
1119                         'priority': 500,
1120                         'mounts': {
1121                             '/tmp': {'kind': 'tmp',
1122                                      "capacity": 1073741824
1123                                  },
1124                             '/var/spool/cwl': {'kind': 'tmp',
1125                                                "capacity": 1073741824 }
1126                         },
1127                         'state': 'Committed',
1128                         'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
1129                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1130                         'output_path': '/var/spool/cwl',
1131                         'output_ttl': 0,
1132                         'container_image': '99999999999999999999999999999993+99',
1133                         'command': ['nvidia-smi'],
1134                         'cwd': '/var/spool/cwl',
1135                         'scheduling_parameters': {},
1136                         'properties': {},
1137                         'secret_mounts': {},
1138                         'output_storage_classes': ["default"]
1139                     }))
1140
1141
1142     # The test passes no builder.resources
1143     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1144     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1145     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1146     def test_match_local_docker(self, keepdocker, determine_image_id):
1147         arvados_cwl.add_arv_hints()
1148         arv_docker_clear_cache()
1149
1150         runner = mock.MagicMock()
1151         runner.ignore_docker_for_reuse = False
1152         runner.intermediate_output_ttl = 0
1153         runner.secret_store = cwltool.secrets.SecretStore()
1154         runner.api._rootDesc = {"revision": "20210628"}
1155
1156         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1157                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1158         determine_image_id.side_effect = lambda x: "123"
1159         def execute(uuid):
1160             ex = mock.MagicMock()
1161             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1162                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1163             ex.execute.return_value = lookup[uuid]
1164             return ex
1165         runner.api.collections().get.side_effect = execute
1166
1167         tool = cmap({
1168             "inputs": [],
1169             "outputs": [],
1170             "baseCommand": "echo",
1171             "arguments": [],
1172             "id": "",
1173             "cwlVersion": "v1.2",
1174             "class": "CommandLineTool"
1175         })
1176
1177         loadingContext, runtimeContext = self.helper(runner, True)
1178
1179         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1180         arvtool.formatgraph = None
1181
1182         container_request = {
1183             'environment': {
1184                 'HOME': '/var/spool/cwl',
1185                 'TMPDIR': '/tmp'
1186             },
1187             'name': 'test_run_True',
1188             'runtime_constraints': {
1189                 'vcpus': 1,
1190                 'ram': 268435456
1191             },
1192             'use_existing': True,
1193             'priority': 500,
1194             'mounts': {
1195                 '/tmp': {'kind': 'tmp',
1196                          "capacity": 1073741824
1197                          },
1198                 '/var/spool/cwl': {'kind': 'tmp',
1199                                    "capacity": 1073741824 }
1200             },
1201             'state': 'Committed',
1202             'output_name': 'Output for step test_run_True',
1203             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1204             'output_path': '/var/spool/cwl',
1205             'output_ttl': 0,
1206             'container_image': '99999999999999999999999999999994+99',
1207             'command': ['echo'],
1208             'cwd': '/var/spool/cwl',
1209             'scheduling_parameters': {},
1210             'properties': {},
1211             'secret_mounts': {},
1212             'output_storage_classes': ["default"]
1213         }
1214
1215         runtimeContext.match_local_docker = False
1216         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1217             j.run(runtimeContext)
1218             runner.api.container_requests().create.assert_called_with(
1219                 body=JsonDiffMatcher(container_request))
1220
1221         arv_docker_clear_cache()
1222         runtimeContext.match_local_docker = True
1223         container_request['container_image'] = '99999999999999999999999999999993+99'
1224         container_request['name'] = 'test_run_True_2'
1225         container_request['output_name'] = 'Output for step test_run_True_2'
1226         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1227             j.run(runtimeContext)
1228             runner.api.container_requests().create.assert_called_with(
1229                 body=JsonDiffMatcher(container_request))
1230
1231
1232
1233 class TestWorkflow(unittest.TestCase):
1234     def setUp(self):
1235         cwltool.process._names = set()
1236         arv_docker_clear_cache()
1237
1238     def helper(self, runner, enable_reuse=True):
1239         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1240
1241         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1242                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1243
1244         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1245         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1246         document_loader.fetch_text = document_loader.fetcher.fetch_text
1247         document_loader.check_exists = document_loader.fetcher.check_exists
1248
1249         loadingContext = arvados_cwl.context.ArvLoadingContext(
1250             {"avsc_names": avsc_names,
1251              "basedir": "",
1252              "make_fs_access": make_fs_access,
1253              "loader": document_loader,
1254              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1255              "construct_tool_object": runner.arv_make_tool})
1256         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1257             {"work_api": "containers",
1258              "basedir": "",
1259              "name": "test_run_wf_"+str(enable_reuse),
1260              "make_fs_access": make_fs_access,
1261              "tmpdir": "/tmp",
1262              "enable_reuse": enable_reuse,
1263              "priority": 500})
1264
1265         return loadingContext, runtimeContext
1266
1267     # The test passes no builder.resources
1268     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1269     @mock.patch("arvados.collection.CollectionReader")
1270     @mock.patch("arvados.collection.Collection")
1271     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1272     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1273         arvados_cwl.add_arv_hints()
1274
1275         api = mock.MagicMock()
1276         api._rootDesc = get_rootDesc()
1277
1278         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1279         self.assertEqual(runner.work_api, 'containers')
1280
1281         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1282         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1283         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1284                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1285
1286         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1287         runner.ignore_docker_for_reuse = False
1288         runner.num_retries = 0
1289         runner.secret_store = cwltool.secrets.SecretStore()
1290
1291         loadingContext, runtimeContext = self.helper(runner)
1292         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1293
1294         mockcollectionreader().exists.return_value = True
1295
1296         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1297         metadata["cwlVersion"] = tool["cwlVersion"]
1298
1299         mockc = mock.MagicMock()
1300         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1301         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1302
1303         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1304         arvtool.formatgraph = None
1305         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1306
1307         next(it).run(runtimeContext)
1308         next(it).run(runtimeContext)
1309
1310         with open("tests/wf/scatter2_subwf.cwl") as f:
1311             subwf = StripYAMLComments(f.read()).rstrip()
1312
1313         runner.api.container_requests().create.assert_called_with(
1314             body=JsonDiffMatcher({
1315                 "command": [
1316                     "cwltool",
1317                     "--no-container",
1318                     "--move-outputs",
1319                     "--preserve-entire-environment",
1320                     "workflow.cwl",
1321                     "cwl.input.yml"
1322                 ],
1323                 "container_image": "99999999999999999999999999999993+99",
1324                 "cwd": "/var/spool/cwl",
1325                 "environment": {
1326                     "HOME": "/var/spool/cwl",
1327                     "TMPDIR": "/tmp"
1328                 },
1329                 "mounts": {
1330                     "/keep/99999999999999999999999999999999+118": {
1331                         "kind": "collection",
1332                         "portable_data_hash": "99999999999999999999999999999999+118"
1333                     },
1334                     "/tmp": {
1335                         "capacity": 1073741824,
1336                         "kind": "tmp"
1337                     },
1338                     "/var/spool/cwl": {
1339                         "capacity": 1073741824,
1340                         "kind": "tmp"
1341                     },
1342                     "/var/spool/cwl/cwl.input.yml": {
1343                         "kind": "collection",
1344                         "path": "cwl.input.yml",
1345                         "portable_data_hash": "99999999999999999999999999999996+99"
1346                     },
1347                     "/var/spool/cwl/workflow.cwl": {
1348                         "kind": "collection",
1349                         "path": "workflow.cwl",
1350                         "portable_data_hash": "99999999999999999999999999999996+99"
1351                     },
1352                     "stdout": {
1353                         "kind": "file",
1354                         "path": "/var/spool/cwl/cwl.output.json"
1355                     }
1356                 },
1357                 "name": "scatterstep",
1358                 "output_name": "Output for step scatterstep",
1359                 "output_path": "/var/spool/cwl",
1360                 "output_ttl": 0,
1361                 "priority": 500,
1362                 "properties": {},
1363                 "runtime_constraints": {
1364                     "ram": 1073741824,
1365                     "vcpus": 1
1366                 },
1367                 "scheduling_parameters": {},
1368                 "secret_mounts": {},
1369                 "state": "Committed",
1370                 "use_existing": True,
1371                 'output_storage_classes': ["default"]
1372             }))
1373         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1374         mockc.open().__enter__().write.assert_has_calls([mock.call(
1375 '''{
1376   "fileblub": {
1377     "basename": "token.txt",
1378     "class": "File",
1379     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1380     "size": 0
1381   },
1382   "sleeptime": 5
1383 }''')])
1384
1385     # The test passes no builder.resources
1386     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1387     @mock.patch("arvados.collection.CollectionReader")
1388     @mock.patch("arvados.collection.Collection")
1389     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1390     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1391         arvados_cwl.add_arv_hints()
1392
1393         api = mock.MagicMock()
1394         api._rootDesc = get_rootDesc()
1395
1396         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1397         self.assertEqual(runner.work_api, 'containers')
1398
1399         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1400         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1401                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1402         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1403                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1404
1405         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1406         runner.ignore_docker_for_reuse = False
1407         runner.num_retries = 0
1408         runner.secret_store = cwltool.secrets.SecretStore()
1409
1410         loadingContext, runtimeContext = self.helper(runner)
1411         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1412         loadingContext.do_update = True
1413         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1414
1415         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1416
1417         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1418         arvtool.formatgraph = None
1419         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1420
1421         next(it).run(runtimeContext)
1422         next(it).run(runtimeContext)
1423
1424         with open("tests/wf/echo-subwf.cwl") as f:
1425             subwf = StripYAMLComments(f.read())
1426
1427         runner.api.container_requests().create.assert_called_with(
1428             body=JsonDiffMatcher({
1429                 'output_ttl': 0,
1430                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1431                 'scheduling_parameters': {},
1432                 'name': u'echo-subwf',
1433                 'secret_mounts': {},
1434                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1435                 'properties': {},
1436                 'priority': 500,
1437                 'mounts': {
1438                     '/var/spool/cwl/cwl.input.yml': {
1439                         'portable_data_hash': '99999999999999999999999999999996+99',
1440                         'kind': 'collection',
1441                         'path': 'cwl.input.yml'
1442                     },
1443                     '/var/spool/cwl/workflow.cwl': {
1444                         'portable_data_hash': '99999999999999999999999999999996+99',
1445                         'kind': 'collection',
1446                         'path': 'workflow.cwl'
1447                     },
1448                     'stdout': {
1449                         'path': '/var/spool/cwl/cwl.output.json',
1450                         'kind': 'file'
1451                     },
1452                     '/tmp': {
1453                         'kind': 'tmp',
1454                         'capacity': 1073741824
1455                     }, '/var/spool/cwl': {
1456                         'kind': 'tmp',
1457                         'capacity': 3221225472
1458                     }
1459                 },
1460                 'state': 'Committed',
1461                 'output_path': '/var/spool/cwl',
1462                 'container_image': '99999999999999999999999999999993+99',
1463                 'command': [
1464                     u'cwltool',
1465                     u'--no-container',
1466                     u'--move-outputs',
1467                     u'--preserve-entire-environment',
1468                     u'workflow.cwl',
1469                     u'cwl.input.yml'
1470                 ],
1471                 'use_existing': True,
1472                 'output_name': u'Output for step echo-subwf',
1473                 'cwd': '/var/spool/cwl',
1474                 'output_storage_classes': ["default"]
1475             }))
1476
1477     def test_default_work_api(self):
1478         arvados_cwl.add_arv_hints()
1479
1480         api = mock.MagicMock()
1481         api._rootDesc = copy.deepcopy(get_rootDesc())
1482         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1483         self.assertEqual(runner.work_api, 'containers')