sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from builtins import str
   6 from builtins import object
   7
   8 import arvados_cwl
   9 import arvados_cwl.context
  10 import arvados_cwl.util
  11 from arvados_cwl.arvdocker import arv_docker_clear_cache
  12 import copy
  13 import arvados.config
  14 import logging
  15 import mock
  16 import unittest
  17 import os
  18 import functools
  19 import cwltool.process
  20 import cwltool.secrets
  21 import cwltool.load_tool
  22 from cwltool.update import INTERNAL_VERSION
  23 from schema_salad.ref_resolver import Loader
  24 from schema_salad.sourceline import cmap
  25
  26 from .matcher import JsonDiffMatcher, StripYAMLComments
  27 from .mock_discovery import get_rootDesc
  28
  29 if not os.getenv('ARVADOS_DEBUG'):
  30     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  31     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  32
  33 class CollectionMock(object):
  34     def __init__(self, vwdmock, *args, **kwargs):
  35         self.vwdmock = vwdmock
  36         self.count = 0
  37
  38     def open(self, *args, **kwargs):
  39         self.count += 1
  40         return self.vwdmock.open(*args, **kwargs)
  41
  42     def copy(self, *args, **kwargs):
  43         self.count += 1
  44         self.vwdmock.copy(*args, **kwargs)
  45
  46     def save_new(self, *args, **kwargs):
  47         pass
  48
  49     def __len__(self):
  50         return self.count
  51
  52     def portable_data_hash(self):
  53         if self.count == 0:
  54             return arvados.config.EMPTY_BLOCK_LOCATOR
  55         else:
  56             return "99999999999999999999999999999996+99"
  57
  58
  59 class TestContainer(unittest.TestCase):
  60
  61     def setUp(self):
  62         cwltool.process._names = set()
  63         arv_docker_clear_cache()
  64
  65     def helper(self, runner, enable_reuse=True):
  66         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
  67
  68         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  69                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  70         fs_access = mock.MagicMock()
  71         fs_access.exists.return_value = True
  72
  73         loadingContext = arvados_cwl.context.ArvLoadingContext(
  74             {"avsc_names": avsc_names,
  75              "basedir": "",
  76              "make_fs_access": make_fs_access,
  77              "construct_tool_object": runner.arv_make_tool,
  78              "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
  79              })
  80         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
  81             {"work_api": "containers",
  82              "basedir": "",
  83              "name": "test_run_"+str(enable_reuse),
  84              "make_fs_access": make_fs_access,
  85              "tmpdir": "/tmp",
  86              "enable_reuse": enable_reuse,
  87              "priority": 500,
  88              "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  89             })
  90
  91         if isinstance(runner, mock.MagicMock):
  92             def make_tool(toolpath_object, loadingContext):
  93                 return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext)
  94             runner.arv_make_tool.side_effect = make_tool
  95
  96         return loadingContext, runtimeContext
  97
  98     # Helper function to set up the ArvCwlExecutor to use the containers api
  99     # and test that the RuntimeStatusLoggingHandler is set up correctly
 100     def setup_and_test_container_executor_and_logging(self, gcc_mock) :
 101         api = mock.MagicMock()
 102         api._rootDesc = copy.deepcopy(get_rootDesc())
 103
 104         # Make sure ArvCwlExecutor thinks it's running inside a container so it
 105         # adds the logging handler that will call runtime_status_update() mock
 106         self.assertFalse(gcc_mock.called)
 107         runner = arvados_cwl.ArvCwlExecutor(api)
 108         self.assertEqual(runner.work_api, 'containers')
 109         root_logger = logging.getLogger('')
 110         handlerClasses = [h.__class__ for h in root_logger.handlers]
 111         self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 112         return runner
 113
 114     # The test passes no builder.resources
 115     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 116     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 117     def test_run(self, keepdocker):
 118         for enable_reuse in (True, False):
 119             arv_docker_clear_cache()
 120
 121             runner = mock.MagicMock()
 122             runner.ignore_docker_for_reuse = False
 123             runner.intermediate_output_ttl = 0
 124             runner.secret_store = cwltool.secrets.SecretStore()
 125             runner.api._rootDesc = {"revision": "20210628"}
 126
 127             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 128             runner.api.collections().get().execute.return_value = {
 129                 "portable_data_hash": "99999999999999999999999999999993+99"}
 130
 131             tool = cmap({
 132                 "inputs": [],
 133                 "outputs": [],
 134                 "baseCommand": "ls",
 135                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 136                 "id": "",
 137                 "class": "CommandLineTool",
 138                 "cwlVersion": "v1.2"
 139             })
 140
 141             loadingContext, runtimeContext = self.helper(runner, enable_reuse)
 142
 143             arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 144             arvtool.formatgraph = None
 145
 146             for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 147                 j.run(runtimeContext)
 148                 runner.api.container_requests().create.assert_called_with(
 149                     body=JsonDiffMatcher({
 150                         'environment': {
 151                             'HOME': '/var/spool/cwl',
 152                             'TMPDIR': '/tmp'
 153                         },
 154                         'name': 'test_run_'+str(enable_reuse),
 155                         'runtime_constraints': {
 156                             'vcpus': 1,
 157                             'ram': 268435456
 158                         },
 159                         'use_existing': enable_reuse,
 160                         'priority': 500,
 161                         'mounts': {
 162                             '/tmp': {'kind': 'tmp',
 163                                      "capacity": 1073741824
 164                                  },
 165                             '/var/spool/cwl': {'kind': 'tmp',
 166                                                "capacity": 1073741824 }
 167                         },
 168                         'state': 'Committed',
 169                         'output_name': 'Output for step test_run_'+str(enable_reuse),
 170                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 171                         'output_path': '/var/spool/cwl',
 172                         'output_ttl': 0,
 173                         'container_image': '99999999999999999999999999999993+99',
 174                         'command': ['ls', '/var/spool/cwl'],
 175                         'cwd': '/var/spool/cwl',
 176                         'scheduling_parameters': {},
 177                         'properties': {},
 178                         'secret_mounts': {},
 179                         'output_storage_classes': ["default"]
 180                     }))
 181
 182     # The test passes some fields in builder.resources
 183     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 184     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 185     def test_resource_requirements(self, keepdocker):
 186         arvados_cwl.add_arv_hints()
 187         runner = mock.MagicMock()
 188         runner.ignore_docker_for_reuse = False
 189         runner.intermediate_output_ttl = 3600
 190         runner.secret_store = cwltool.secrets.SecretStore()
 191         runner.api._rootDesc = {"revision": "20210628"}
 192
 193         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 194         runner.api.collections().get().execute.return_value = {
 195             "portable_data_hash": "99999999999999999999999999999993+99"}
 196
 197         tool = cmap({
 198             "inputs": [],
 199             "outputs": [],
 200             "hints": [{
 201                 "class": "ResourceRequirement",
 202                 "coresMin": 3,
 203                 "ramMin": 3000,
 204                 "tmpdirMin": 4000,
 205                 "outdirMin": 5000
 206             }, {
 207                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 208                 "keep_cache": 512
 209             }, {
 210                 "class": "http://arvados.org/cwl#APIRequirement",
 211             }, {
 212                 "class": "http://arvados.org/cwl#PartitionRequirement",
 213                 "partition": "blurb"
 214             }, {
 215                 "class": "http://arvados.org/cwl#IntermediateOutput",
 216                 "outputTTL": 7200
 217             }, {
 218                 "class": "WorkReuse",
 219                 "enableReuse": False
 220             }],
 221             "baseCommand": "ls",
 222             "id": "",
 223             "class": "CommandLineTool",
 224             "cwlVersion": "v1.2"
 225         })
 226
 227         loadingContext, runtimeContext = self.helper(runner)
 228         runtimeContext.name = "test_resource_requirements"
 229
 230         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 231         arvtool.formatgraph = None
 232         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 233             j.run(runtimeContext)
 234
 235         call_args, call_kwargs = runner.api.container_requests().create.call_args
 236
 237         call_body_expected = {
 238             'environment': {
 239                 'HOME': '/var/spool/cwl',
 240                 'TMPDIR': '/tmp'
 241             },
 242             'name': 'test_resource_requirements',
 243             'runtime_constraints': {
 244                 'vcpus': 3,
 245                 'ram': 3145728000,
 246                 'keep_cache_ram': 536870912,
 247                 'API': True
 248             },
 249             'use_existing': False,
 250             'priority': 500,
 251             'mounts': {
 252                 '/tmp': {'kind': 'tmp',
 253                          "capacity": 4194304000 },
 254                 '/var/spool/cwl': {'kind': 'tmp',
 255                                    "capacity": 5242880000 }
 256             },
 257             'state': 'Committed',
 258             'output_name': 'Output for step test_resource_requirements',
 259             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 260             'output_path': '/var/spool/cwl',
 261             'output_ttl': 7200,
 262             'container_image': '99999999999999999999999999999993+99',
 263             'command': ['ls'],
 264             'cwd': '/var/spool/cwl',
 265             'scheduling_parameters': {
 266                 'partitions': ['blurb']
 267             },
 268             'properties': {},
 269             'secret_mounts': {},
 270             'output_storage_classes': ["default"]
 271         }
 272
 273         call_body = call_kwargs.get('body', None)
 274         self.assertNotEqual(None, call_body)
 275         for key in call_body:
 276             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 277
 278
 279     # The test passes some fields in builder.resources
 280     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 281     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 282     @mock.patch("arvados.collection.Collection")
 283     def test_initial_work_dir(self, collection_mock, keepdocker):
 284         runner = mock.MagicMock()
 285         runner.ignore_docker_for_reuse = False
 286         runner.intermediate_output_ttl = 0
 287         runner.secret_store = cwltool.secrets.SecretStore()
 288         runner.api._rootDesc = {"revision": "20210628"}
 289
 290         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 291         runner.api.collections().get().execute.return_value = {
 292             "portable_data_hash": "99999999999999999999999999999993+99"}
 293
 294         sourcemock = mock.MagicMock()
 295         def get_collection_mock(p):
 296             if "/" in p:
 297                 return (sourcemock, p.split("/", 1)[1])
 298             else:
 299                 return (sourcemock, "")
 300         runner.fs_access.get_collection.side_effect = get_collection_mock
 301
 302         vwdmock = mock.MagicMock()
 303         collection_mock.side_effect = lambda *args, **kwargs: CollectionMock(vwdmock, *args, **kwargs)
 304
 305         tool = cmap({
 306             "inputs": [],
 307             "outputs": [],
 308             "hints": [{
 309                 "class": "InitialWorkDirRequirement",
 310                 "listing": [{
 311                     "class": "File",
 312                     "basename": "foo",
 313                     "location": "keep:99999999999999999999999999999995+99/bar"
 314                 },
 315                 {
 316                     "class": "Directory",
 317                     "basename": "foo2",
 318                     "location": "keep:99999999999999999999999999999995+99"
 319                 },
 320                 {
 321                     "class": "File",
 322                     "basename": "filename",
 323                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 324                 },
 325                 {
 326                     "class": "Directory",
 327                     "basename": "subdir",
 328                     "location": "keep:99999999999999999999999999999995+99/subdir"
 329                 }                        ]
 330             }],
 331             "baseCommand": "ls",
 332             "class": "CommandLineTool",
 333             "cwlVersion": "v1.2",
 334             "id": ""
 335         })
 336
 337         loadingContext, runtimeContext = self.helper(runner)
 338         runtimeContext.name = "test_initial_work_dir"
 339
 340         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 341
 342         arvtool.formatgraph = None
 343         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 344             j.run(runtimeContext)
 345
 346         call_args, call_kwargs = runner.api.container_requests().create.call_args
 347
 348         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 349         vwdmock.copy.assert_has_calls([mock.call('.', 'foo2', source_collection=sourcemock)])
 350         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 351         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 352
 353         call_body_expected = {
 354             'environment': {
 355                 'HOME': '/var/spool/cwl',
 356                 'TMPDIR': '/tmp'
 357             },
 358             'name': 'test_initial_work_dir',
 359             'runtime_constraints': {
 360                 'vcpus': 1,
 361                 'ram': 268435456
 362             },
 363             'use_existing': True,
 364             'priority': 500,
 365             'mounts': {
 366                 '/tmp': {'kind': 'tmp',
 367                          "capacity": 1073741824 },
 368                 '/var/spool/cwl': {'kind': 'tmp',
 369                                    "capacity": 1073741824 },
 370                 '/var/spool/cwl/foo': {
 371                     'kind': 'collection',
 372                     'path': 'foo',
 373                     'portable_data_hash': '99999999999999999999999999999996+99'
 374                 },
 375                 '/var/spool/cwl/foo2': {
 376                     'kind': 'collection',
 377                     'path': 'foo2',
 378                     'portable_data_hash': '99999999999999999999999999999996+99'
 379                 },
 380                 '/var/spool/cwl/filename': {
 381                     'kind': 'collection',
 382                     'path': 'filename',
 383                     'portable_data_hash': '99999999999999999999999999999996+99'
 384                 },
 385                 '/var/spool/cwl/subdir': {
 386                     'kind': 'collection',
 387                     'path': 'subdir',
 388                     'portable_data_hash': '99999999999999999999999999999996+99'
 389                 }
 390             },
 391             'state': 'Committed',
 392             'output_name': 'Output for step test_initial_work_dir',
 393             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 394             'output_path': '/var/spool/cwl',
 395             'output_ttl': 0,
 396             'container_image': '99999999999999999999999999999993+99',
 397             'command': ['ls'],
 398             'cwd': '/var/spool/cwl',
 399             'scheduling_parameters': {
 400             },
 401             'properties': {},
 402             'secret_mounts': {},
 403             'output_storage_classes': ["default"]
 404         }
 405
 406         call_body = call_kwargs.get('body', None)
 407         self.assertNotEqual(None, call_body)
 408         for key in call_body:
 409             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 410
 411
 412     # Test redirecting stdin/stdout/stderr
 413     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 414     def test_redirects(self, keepdocker):
 415         runner = mock.MagicMock()
 416         runner.ignore_docker_for_reuse = False
 417         runner.intermediate_output_ttl = 0
 418         runner.secret_store = cwltool.secrets.SecretStore()
 419         runner.api._rootDesc = {"revision": "20210628"}
 420
 421         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 422         runner.api.collections().get().execute.return_value = {
 423             "portable_data_hash": "99999999999999999999999999999993+99"}
 424
 425         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 426
 427         tool = cmap({
 428             "inputs": [],
 429             "outputs": [],
 430             "baseCommand": "ls",
 431             "stdout": "stdout.txt",
 432             "stderr": "stderr.txt",
 433             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 434             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 435             "id": "",
 436             "class": "CommandLineTool",
 437             "cwlVersion": "v1.2"
 438         })
 439
 440         loadingContext, runtimeContext = self.helper(runner)
 441         runtimeContext.name = "test_run_redirect"
 442
 443         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 444         arvtool.formatgraph = None
 445         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 446             j.run(runtimeContext)
 447             runner.api.container_requests().create.assert_called_with(
 448                 body=JsonDiffMatcher({
 449                     'environment': {
 450                         'HOME': '/var/spool/cwl',
 451                         'TMPDIR': '/tmp'
 452                     },
 453                     'name': 'test_run_redirect',
 454                     'runtime_constraints': {
 455                         'vcpus': 1,
 456                         'ram': 268435456
 457                     },
 458                     'use_existing': True,
 459                     'priority': 500,
 460                     'mounts': {
 461                         '/tmp': {'kind': 'tmp',
 462                                  "capacity": 1073741824 },
 463                         '/var/spool/cwl': {'kind': 'tmp',
 464                                            "capacity": 1073741824 },
 465                         "stderr": {
 466                             "kind": "file",
 467                             "path": "/var/spool/cwl/stderr.txt"
 468                         },
 469                         "stdin": {
 470                             "kind": "collection",
 471                             "path": "file.txt",
 472                             "portable_data_hash": "99999999999999999999999999999996+99"
 473                         },
 474                         "stdout": {
 475                             "kind": "file",
 476                             "path": "/var/spool/cwl/stdout.txt"
 477                         },
 478                     },
 479                     'state': 'Committed',
 480                     "output_name": "Output for step test_run_redirect",
 481                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 482                     'output_path': '/var/spool/cwl',
 483                     'output_ttl': 0,
 484                     'container_image': '99999999999999999999999999999993+99',
 485                     'command': ['ls', '/var/spool/cwl'],
 486                     'cwd': '/var/spool/cwl',
 487                     'scheduling_parameters': {},
 488                     'properties': {},
 489                     'secret_mounts': {},
 490                     'output_storage_classes': ["default"]
 491                 }))
 492
 493     @mock.patch("arvados.collection.Collection")
 494     def test_done(self, col):
 495         api = mock.MagicMock()
 496
 497         runner = mock.MagicMock()
 498         runner.api = api
 499         runner.num_retries = 0
 500         runner.ignore_docker_for_reuse = False
 501         runner.intermediate_output_ttl = 0
 502         runner.secret_store = cwltool.secrets.SecretStore()
 503
 504         runner.api.containers().get().execute.return_value = {"state":"Complete",
 505                                                               "output": "abc+123",
 506                                                               "exit_code": 0}
 507
 508         col().open.return_value = []
 509
 510         loadingContext, runtimeContext = self.helper(runner)
 511
 512         arvjob = arvados_cwl.ArvadosContainer(runner,
 513                                               runtimeContext,
 514                                               mock.MagicMock(),
 515                                               {},
 516                                               None,
 517                                               [],
 518                                               [],
 519                                               "testjob")
 520         arvjob.output_callback = mock.MagicMock()
 521         arvjob.collect_outputs = mock.MagicMock()
 522         arvjob.successCodes = [0]
 523         arvjob.outdir = "/var/spool/cwl"
 524         arvjob.output_ttl = 3600
 525
 526         arvjob.collect_outputs.return_value = {"out": "stuff"}
 527
 528         arvjob.done({
 529             "state": "Final",
 530             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 531             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 532             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 533             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 534             "modified_at": "2017-05-26T12:01:22Z"
 535         })
 536
 537         self.assertFalse(api.collections().create.called)
 538         self.assertFalse(runner.runtime_status_error.called)
 539
 540         arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
 541         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 542         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 543
 544     # Test to make sure we dont call runtime_status_update if we already did
 545     # some where higher up in the call stack
 546     @mock.patch("arvados_cwl.util.get_current_container")
 547     def test_recursive_runtime_status_update(self, gcc_mock):
 548         self.setup_and_test_container_executor_and_logging(gcc_mock)
 549         root_logger = logging.getLogger('')
 550
 551         # get_current_container is invoked when we call runtime_status_update
 552         # so try and log again!
 553         gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
 554         try:
 555             root_logger.error("First Error")
 556         except RuntimeError:
 557             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 558
 559
 560     # Test to make sure that an exception raised from
 561     # get_current_container doesn't cause the logger to raise an
 562     # exception
 563     @mock.patch("arvados_cwl.util.get_current_container")
 564     def test_runtime_status_get_current_container_exception(self, gcc_mock):
 565         self.setup_and_test_container_executor_and_logging(gcc_mock)
 566         root_logger = logging.getLogger('')
 567
 568         # get_current_container is invoked when we call
 569         # runtime_status_update, it is going to also raise an
 570         # exception.
 571         gcc_mock.side_effect = Exception("Second Error")
 572         try:
 573             root_logger.error("First Error")
 574         except Exception:
 575             self.fail("Exception in logger should not propagate")
 576         self.assertTrue(gcc_mock.called)
 577
 578     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
 579     @mock.patch("arvados_cwl.util.get_current_container")
 580     @mock.patch("arvados.collection.CollectionReader")
 581     @mock.patch("arvados.collection.Collection")
 582     def test_child_failure(self, col, reader, gcc_mock, rts_mock):
 583         runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 584
 585         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
 586         self.assertTrue(gcc_mock.called)
 587
 588         runner.num_retries = 0
 589         runner.ignore_docker_for_reuse = False
 590         runner.intermediate_output_ttl = 0
 591         runner.secret_store = cwltool.secrets.SecretStore()
 592         runner.label = mock.MagicMock()
 593         runner.label.return_value = '[container testjob]'
 594
 595         runner.api.containers().get().execute.return_value = {
 596             "state":"Complete",
 597             "output": "abc+123",
 598             "exit_code": 1,
 599             "log": "def+234"
 600         }
 601
 602         col().open.return_value = []
 603
 604         loadingContext, runtimeContext = self.helper(runner)
 605
 606         arvjob = arvados_cwl.ArvadosContainer(runner,
 607                                               runtimeContext,
 608                                               mock.MagicMock(),
 609                                               {},
 610                                               None,
 611                                               [],
 612                                               [],
 613                                               "testjob")
 614         arvjob.output_callback = mock.MagicMock()
 615         arvjob.collect_outputs = mock.MagicMock()
 616         arvjob.successCodes = [0]
 617         arvjob.outdir = "/var/spool/cwl"
 618         arvjob.output_ttl = 3600
 619         arvjob.collect_outputs.return_value = {"out": "stuff"}
 620
 621         arvjob.done({
 622             "state": "Final",
 623             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 624             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 625             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 626             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 627             "modified_at": "2017-05-26T12:01:22Z"
 628         })
 629
 630         rts_mock.assert_called_with(
 631             'error',
 632             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
 633             '  ** log is empty **'
 634         )
 635         arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
 636
 637     # The test passes no builder.resources
 638     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 639     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 640     def test_mounts(self, keepdocker):
 641         runner = mock.MagicMock()
 642         runner.ignore_docker_for_reuse = False
 643         runner.intermediate_output_ttl = 0
 644         runner.secret_store = cwltool.secrets.SecretStore()
 645         runner.api._rootDesc = {"revision": "20210628"}
 646
 647         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 648         runner.api.collections().get().execute.return_value = {
 649             "portable_data_hash": "99999999999999999999999999999994+99",
 650             "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 651
 652         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 653
 654         tool = cmap({
 655             "inputs": [
 656                 {"id": "p1",
 657                  "type": "Directory"}
 658             ],
 659             "outputs": [],
 660             "baseCommand": "ls",
 661             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 662             "id": "",
 663             "class": "CommandLineTool",
 664             "cwlVersion": "v1.2"
 665         })
 666
 667         loadingContext, runtimeContext = self.helper(runner)
 668         runtimeContext.name = "test_run_mounts"
 669
 670         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 671         arvtool.formatgraph = None
 672         job_order = {
 673             "p1": {
 674                 "class": "Directory",
 675                 "location": "keep:99999999999999999999999999999994+44",
 676                 "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
 677                 "listing": [
 678                     {
 679                         "class": "File",
 680                         "location": "keep:99999999999999999999999999999994+44/file1",
 681                     },
 682                     {
 683                         "class": "File",
 684                         "location": "keep:99999999999999999999999999999994+44/file2",
 685                     }
 686                 ]
 687             }
 688         }
 689         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 690             j.run(runtimeContext)
 691             runner.api.container_requests().create.assert_called_with(
 692                 body=JsonDiffMatcher({
 693                     'environment': {
 694                         'HOME': '/var/spool/cwl',
 695                         'TMPDIR': '/tmp'
 696                     },
 697                     'name': 'test_run_mounts',
 698                     'runtime_constraints': {
 699                         'vcpus': 1,
 700                         'ram': 268435456
 701                     },
 702                     'use_existing': True,
 703                     'priority': 500,
 704                     'mounts': {
 705                         "/keep/99999999999999999999999999999994+44": {
 706                             "kind": "collection",
 707                             "portable_data_hash": "99999999999999999999999999999994+44",
 708                             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 709                         },
 710                         '/tmp': {'kind': 'tmp',
 711                                  "capacity": 1073741824 },
 712                         '/var/spool/cwl': {'kind': 'tmp',
 713                                            "capacity": 1073741824 }
 714                     },
 715                     'state': 'Committed',
 716                     'output_name': 'Output for step test_run_mounts',
 717                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 718                     'output_path': '/var/spool/cwl',
 719                     'output_ttl': 0,
 720                     'container_image': '99999999999999999999999999999994+99',
 721                     'command': ['ls', '/var/spool/cwl'],
 722                     'cwd': '/var/spool/cwl',
 723                     'scheduling_parameters': {},
 724                     'properties': {},
 725                     'secret_mounts': {},
 726                     'output_storage_classes': ["default"]
 727                 }))
 728
 729     # The test passes no builder.resources
 730     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 731     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 732     def test_secrets(self, keepdocker):
 733         arvados_cwl.add_arv_hints()
 734         runner = mock.MagicMock()
 735         runner.ignore_docker_for_reuse = False
 736         runner.intermediate_output_ttl = 0
 737         runner.secret_store = cwltool.secrets.SecretStore()
 738         runner.api._rootDesc = {"revision": "20210628"}
 739
 740         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 741         runner.api.collections().get().execute.return_value = {
 742             "portable_data_hash": "99999999999999999999999999999993+99"}
 743
 744         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
 745
 746         tool = cmap({"arguments": ["md5sum", "example.conf"],
 747                      "class": "CommandLineTool",
 748                      "cwlVersion": "v1.2",
 749                      "hints": [
 750                          {
 751                              "class": "http://commonwl.org/cwltool#Secrets",
 752                              "secrets": [
 753                                  "#secret_job.cwl/pw"
 754                              ]
 755                          }
 756                      ],
 757                      "id": "",
 758                      "inputs": [
 759                          {
 760                              "id": "#secret_job.cwl/pw",
 761                              "type": "string"
 762                          }
 763                      ],
 764                      "outputs": [
 765                      ],
 766                      "requirements": [
 767                          {
 768                              "class": "InitialWorkDirRequirement",
 769                              "listing": [
 770                                  {
 771                                      "entry": "username: user\npassword: $(inputs.pw)\n",
 772                                      "entryname": "example.conf"
 773                                  }
 774                              ]
 775                          }
 776                      ]})
 777
 778         loadingContext, runtimeContext = self.helper(runner)
 779         runtimeContext.name = "test_secrets"
 780
 781         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 782         arvtool.formatgraph = None
 783
 784         job_order = {"pw": "blorp"}
 785         runner.secret_store.store(["pw"], job_order)
 786
 787         for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext):
 788             j.run(runtimeContext)
 789             runner.api.container_requests().create.assert_called_with(
 790                 body=JsonDiffMatcher({
 791                     'environment': {
 792                         'HOME': '/var/spool/cwl',
 793                         'TMPDIR': '/tmp'
 794                     },
 795                     'name': 'test_secrets',
 796                     'runtime_constraints': {
 797                         'vcpus': 1,
 798                         'ram': 268435456
 799                     },
 800                     'use_existing': True,
 801                     'priority': 500,
 802                     'mounts': {
 803                         '/tmp': {'kind': 'tmp',
 804                                  "capacity": 1073741824
 805                              },
 806                         '/var/spool/cwl': {'kind': 'tmp',
 807                                            "capacity": 1073741824 }
 808                     },
 809                     'state': 'Committed',
 810                     'output_name': 'Output for step test_secrets',
 811                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 812                     'output_path': '/var/spool/cwl',
 813                     'output_ttl': 0,
 814                     'container_image': '99999999999999999999999999999993+99',
 815                     'command': ['md5sum', 'example.conf'],
 816                     'cwd': '/var/spool/cwl',
 817                     'scheduling_parameters': {},
 818                     'properties': {},
 819                     "secret_mounts": {
 820                         "/var/spool/cwl/example.conf": {
 821                             "content": "username: user\npassword: blorp\n",
 822                             "kind": "text"
 823                         }
 824                     },
 825                     'output_storage_classes': ["default"]
 826                 }))
 827
 828     # The test passes no builder.resources
 829     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 830     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 831     def test_timelimit(self, keepdocker):
 832         runner = mock.MagicMock()
 833         runner.ignore_docker_for_reuse = False
 834         runner.intermediate_output_ttl = 0
 835         runner.secret_store = cwltool.secrets.SecretStore()
 836         runner.api._rootDesc = {"revision": "20210628"}
 837
 838         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 839         runner.api.collections().get().execute.return_value = {
 840             "portable_data_hash": "99999999999999999999999999999993+99"}
 841
 842         tool = cmap({
 843             "inputs": [],
 844             "outputs": [],
 845             "baseCommand": "ls",
 846             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 847             "id": "",
 848             "cwlVersion": "v1.2",
 849             "class": "CommandLineTool",
 850             "hints": [
 851                 {
 852                     "class": "ToolTimeLimit",
 853                     "timelimit": 42
 854                 }
 855             ]
 856         })
 857
 858         loadingContext, runtimeContext = self.helper(runner)
 859         runtimeContext.name = "test_timelimit"
 860
 861         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 862         arvtool.formatgraph = None
 863
 864         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 865             j.run(runtimeContext)
 866
 867         _, kwargs = runner.api.container_requests().create.call_args
 868         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
 869
 870
 871     # The test passes no builder.resources
 872     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 873     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 874     def test_setting_storage_class(self, keepdocker):
 875         arv_docker_clear_cache()
 876
 877         runner = mock.MagicMock()
 878         runner.ignore_docker_for_reuse = False
 879         runner.intermediate_output_ttl = 0
 880         runner.secret_store = cwltool.secrets.SecretStore()
 881         runner.api._rootDesc = {"revision": "20210628"}
 882
 883         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 884         runner.api.collections().get().execute.return_value = {
 885             "portable_data_hash": "99999999999999999999999999999993+99"}
 886
 887         tool = cmap({
 888             "inputs": [],
 889             "outputs": [],
 890             "baseCommand": "ls",
 891             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 892             "id": "",
 893             "cwlVersion": "v1.2",
 894             "class": "CommandLineTool",
 895             "hints": [
 896                 {
 897                     "class": "http://arvados.org/cwl#OutputStorageClass",
 898                     "finalStorageClass": ["baz_sc", "qux_sc"],
 899                     "intermediateStorageClass": ["foo_sc", "bar_sc"]
 900                 }
 901             ]
 902         })
 903
 904         loadingContext, runtimeContext = self.helper(runner, True)
 905
 906         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 907         arvtool.formatgraph = None
 908
 909         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
 910             j.run(runtimeContext)
 911             runner.api.container_requests().create.assert_called_with(
 912                 body=JsonDiffMatcher({
 913                     'environment': {
 914                         'HOME': '/var/spool/cwl',
 915                         'TMPDIR': '/tmp'
 916                     },
 917                     'name': 'test_run_True',
 918                     'runtime_constraints': {
 919                         'vcpus': 1,
 920                         'ram': 268435456
 921                     },
 922                     'use_existing': True,
 923                     'priority': 500,
 924                     'mounts': {
 925                         '/tmp': {'kind': 'tmp',
 926                                  "capacity": 1073741824
 927                              },
 928                         '/var/spool/cwl': {'kind': 'tmp',
 929                                            "capacity": 1073741824 }
 930                     },
 931                     'state': 'Committed',
 932                     'output_name': 'Output for step test_run_True',
 933                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 934                     'output_path': '/var/spool/cwl',
 935                     'output_ttl': 0,
 936                     'container_image': '99999999999999999999999999999993+99',
 937                     'command': ['ls', '/var/spool/cwl'],
 938                     'cwd': '/var/spool/cwl',
 939                     'scheduling_parameters': {},
 940                     'properties': {},
 941                     'secret_mounts': {},
 942                     'output_storage_classes': ["foo_sc", "bar_sc"]
 943                 }))
 944
 945
 946     # The test passes no builder.resources
 947     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 948     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 949     def test_setting_process_properties(self, keepdocker):
 950         arv_docker_clear_cache()
 951
 952         runner = mock.MagicMock()
 953         runner.ignore_docker_for_reuse = False
 954         runner.intermediate_output_ttl = 0
 955         runner.secret_store = cwltool.secrets.SecretStore()
 956         runner.api._rootDesc = {"revision": "20210628"}
 957
 958         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 959         runner.api.collections().get().execute.return_value = {
 960             "portable_data_hash": "99999999999999999999999999999993+99"}
 961
 962         tool = cmap({
 963             "inputs": [
 964                 {"id": "x", "type": "string"}],
 965             "outputs": [],
 966             "baseCommand": "ls",
 967             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 968             "id": "",
 969             "class": "CommandLineTool",
 970             "cwlVersion": "v1.2",
 971             "hints": [
 972             {
 973                 "class": "http://arvados.org/cwl#ProcessProperties",
 974                 "processProperties": [
 975                     {"propertyName": "foo",
 976                      "propertyValue": "bar"},
 977                     {"propertyName": "baz",
 978                      "propertyValue": "$(inputs.x)"},
 979                     {"propertyName": "quux",
 980                      "propertyValue": {
 981                          "q1": 1,
 982                          "q2": 2
 983                      }
 984                     }
 985                 ],
 986             }
 987         ]
 988         })
 989
 990         loadingContext, runtimeContext = self.helper(runner, True)
 991
 992         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
 993         arvtool.formatgraph = None
 994
 995         for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
 996             j.run(runtimeContext)
 997             runner.api.container_requests().create.assert_called_with(
 998                 body=JsonDiffMatcher({
 999                     'environment': {
1000                         'HOME': '/var/spool/cwl',
1001                         'TMPDIR': '/tmp'
1002                     },
1003                     'name': 'test_run_True',
1004                     'runtime_constraints': {
1005                         'vcpus': 1,
1006                         'ram': 268435456
1007                     },
1008                     'use_existing': True,
1009                     'priority': 500,
1010                     'mounts': {
1011                         '/tmp': {'kind': 'tmp',
1012                                  "capacity": 1073741824
1013                              },
1014                         '/var/spool/cwl': {'kind': 'tmp',
1015                                            "capacity": 1073741824 }
1016                     },
1017                     'state': 'Committed',
1018                     'output_name': 'Output for step test_run_True',
1019                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1020                     'output_path': '/var/spool/cwl',
1021                     'output_ttl': 0,
1022                     'container_image': '99999999999999999999999999999993+99',
1023                     'command': ['ls', '/var/spool/cwl'],
1024                     'cwd': '/var/spool/cwl',
1025                     'scheduling_parameters': {},
1026                     'properties': {
1027                         "baz": "blorp",
1028                         "foo": "bar",
1029                         "quux": {
1030                             "q1": 1,
1031                             "q2": 2
1032                         }
1033                     },
1034                     'secret_mounts': {},
1035                     'output_storage_classes': ["default"]
1036                 }))
1037
1038
1039     # The test passes no builder.resources
1040     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1041     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1042     def test_cuda_requirement(self, keepdocker):
1043         arvados_cwl.add_arv_hints()
1044         arv_docker_clear_cache()
1045
1046         runner = mock.MagicMock()
1047         runner.ignore_docker_for_reuse = False
1048         runner.intermediate_output_ttl = 0
1049         runner.secret_store = cwltool.secrets.SecretStore()
1050         runner.api._rootDesc = {"revision": "20210628"}
1051
1052         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
1053         runner.api.collections().get().execute.return_value = {
1054             "portable_data_hash": "99999999999999999999999999999993+99"}
1055
1056         tool = cmap({
1057             "inputs": [],
1058             "outputs": [],
1059             "baseCommand": "nvidia-smi",
1060             "arguments": [],
1061             "id": "",
1062             "cwlVersion": "v1.2",
1063             "class": "CommandLineTool",
1064             "requirements": [
1065             {
1066                 "class": "http://commonwl.org/cwltool#CUDARequirement",
1067                 "cudaVersionMin": "11.0",
1068                 "cudaComputeCapabilityMin": "9.0",
1069             }
1070         ]
1071         })
1072
1073         loadingContext, runtimeContext = self.helper(runner, True)
1074
1075         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1076         arvtool.formatgraph = None
1077
1078         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1079             j.run(runtimeContext)
1080             runner.api.container_requests().create.assert_called_with(
1081                 body=JsonDiffMatcher({
1082                     'environment': {
1083                         'HOME': '/var/spool/cwl',
1084                         'TMPDIR': '/tmp'
1085                     },
1086                     'name': 'test_run_True',
1087                     'runtime_constraints': {
1088                         'vcpus': 1,
1089                         'ram': 268435456,
1090                         'cuda': {
1091                             'device_count': 1,
1092                             'driver_version': "11.0",
1093                             'hardware_capability': "9.0"
1094                         }
1095                     },
1096                     'use_existing': True,
1097                     'priority': 500,
1098                     'mounts': {
1099                         '/tmp': {'kind': 'tmp',
1100                                  "capacity": 1073741824
1101                              },
1102                         '/var/spool/cwl': {'kind': 'tmp',
1103                                            "capacity": 1073741824 }
1104                     },
1105                     'state': 'Committed',
1106                     'output_name': 'Output for step test_run_True',
1107                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1108                     'output_path': '/var/spool/cwl',
1109                     'output_ttl': 0,
1110                     'container_image': '99999999999999999999999999999993+99',
1111                     'command': ['nvidia-smi'],
1112                     'cwd': '/var/spool/cwl',
1113                     'scheduling_parameters': {},
1114                     'properties': {},
1115                     'secret_mounts': {},
1116                     'output_storage_classes': ["default"]
1117                 }))
1118
1119
1120     # The test passes no builder.resources
1121     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1122     @mock.patch("arvados_cwl.arvdocker.determine_image_id")
1123     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
1124     def test_match_local_docker(self, keepdocker, determine_image_id):
1125         arvados_cwl.add_arv_hints()
1126         arv_docker_clear_cache()
1127
1128         runner = mock.MagicMock()
1129         runner.ignore_docker_for_reuse = False
1130         runner.intermediate_output_ttl = 0
1131         runner.secret_store = cwltool.secrets.SecretStore()
1132         runner.api._rootDesc = {"revision": "20210628"}
1133
1134         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
1135                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
1136         determine_image_id.side_effect = lambda x: "123"
1137         def execute(uuid):
1138             ex = mock.MagicMock()
1139             lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
1140                       "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
1141             ex.execute.return_value = lookup[uuid]
1142             return ex
1143         runner.api.collections().get.side_effect = execute
1144
1145         tool = cmap({
1146             "inputs": [],
1147             "outputs": [],
1148             "baseCommand": "echo",
1149             "arguments": [],
1150             "id": "",
1151             "cwlVersion": "v1.2",
1152             "class": "CommandLineTool"
1153         })
1154
1155         loadingContext, runtimeContext = self.helper(runner, True)
1156
1157         arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
1158         arvtool.formatgraph = None
1159
1160         container_request = {
1161             'environment': {
1162                 'HOME': '/var/spool/cwl',
1163                 'TMPDIR': '/tmp'
1164             },
1165             'name': 'test_run_True',
1166             'runtime_constraints': {
1167                 'vcpus': 1,
1168                 'ram': 268435456
1169             },
1170             'use_existing': True,
1171             'priority': 500,
1172             'mounts': {
1173                 '/tmp': {'kind': 'tmp',
1174                          "capacity": 1073741824
1175                          },
1176                 '/var/spool/cwl': {'kind': 'tmp',
1177                                    "capacity": 1073741824 }
1178             },
1179             'state': 'Committed',
1180             'output_name': 'Output for step test_run_True',
1181             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
1182             'output_path': '/var/spool/cwl',
1183             'output_ttl': 0,
1184             'container_image': '99999999999999999999999999999994+99',
1185             'command': ['echo'],
1186             'cwd': '/var/spool/cwl',
1187             'scheduling_parameters': {},
1188             'properties': {},
1189             'secret_mounts': {},
1190             'output_storage_classes': ["default"]
1191         }
1192
1193         runtimeContext.match_local_docker = False
1194         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1195             j.run(runtimeContext)
1196             runner.api.container_requests().create.assert_called_with(
1197                 body=JsonDiffMatcher(container_request))
1198
1199         arv_docker_clear_cache()
1200         runtimeContext.match_local_docker = True
1201         container_request['container_image'] = '99999999999999999999999999999993+99'
1202         container_request['name'] = 'test_run_True_2'
1203         container_request['output_name'] = 'Output for step test_run_True_2'
1204         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
1205             j.run(runtimeContext)
1206             runner.api.container_requests().create.assert_called_with(
1207                 body=JsonDiffMatcher(container_request))
1208
1209
1210
1211 class TestWorkflow(unittest.TestCase):
1212     def setUp(self):
1213         cwltool.process._names = set()
1214         arv_docker_clear_cache()
1215
1216     def helper(self, runner, enable_reuse=True):
1217         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
1218
1219         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
1220                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
1221
1222         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
1223         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
1224         document_loader.fetch_text = document_loader.fetcher.fetch_text
1225         document_loader.check_exists = document_loader.fetcher.check_exists
1226
1227         loadingContext = arvados_cwl.context.ArvLoadingContext(
1228             {"avsc_names": avsc_names,
1229              "basedir": "",
1230              "make_fs_access": make_fs_access,
1231              "loader": document_loader,
1232              "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
1233              "construct_tool_object": runner.arv_make_tool})
1234         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
1235             {"work_api": "containers",
1236              "basedir": "",
1237              "name": "test_run_wf_"+str(enable_reuse),
1238              "make_fs_access": make_fs_access,
1239              "tmpdir": "/tmp",
1240              "enable_reuse": enable_reuse,
1241              "priority": 500})
1242
1243         return loadingContext, runtimeContext
1244
1245     # The test passes no builder.resources
1246     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1247     @mock.patch("arvados.collection.CollectionReader")
1248     @mock.patch("arvados.collection.Collection")
1249     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1250     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
1251         arvados_cwl.add_arv_hints()
1252
1253         api = mock.MagicMock()
1254         api._rootDesc = get_rootDesc()
1255
1256         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1257         self.assertEqual(runner.work_api, 'containers')
1258
1259         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1260         runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"}
1261         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1262                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1263
1264         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1265         runner.ignore_docker_for_reuse = False
1266         runner.num_retries = 0
1267         runner.secret_store = cwltool.secrets.SecretStore()
1268
1269         loadingContext, runtimeContext = self.helper(runner)
1270         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1271
1272         mockcollectionreader().exists.return_value = True
1273
1274         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
1275         metadata["cwlVersion"] = tool["cwlVersion"]
1276
1277         mockc = mock.MagicMock()
1278         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mockc, *args, **kwargs)
1279         mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
1280
1281         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1282         arvtool.formatgraph = None
1283         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1284
1285         next(it).run(runtimeContext)
1286         next(it).run(runtimeContext)
1287
1288         with open("tests/wf/scatter2_subwf.cwl") as f:
1289             subwf = StripYAMLComments(f.read()).rstrip()
1290
1291         runner.api.container_requests().create.assert_called_with(
1292             body=JsonDiffMatcher({
1293                 "command": [
1294                     "cwltool",
1295                     "--no-container",
1296                     "--move-outputs",
1297                     "--preserve-entire-environment",
1298                     "workflow.cwl",
1299                     "cwl.input.yml"
1300                 ],
1301                 "container_image": "99999999999999999999999999999993+99",
1302                 "cwd": "/var/spool/cwl",
1303                 "environment": {
1304                     "HOME": "/var/spool/cwl",
1305                     "TMPDIR": "/tmp"
1306                 },
1307                 "mounts": {
1308                     "/keep/99999999999999999999999999999999+118": {
1309                         "kind": "collection",
1310                         "portable_data_hash": "99999999999999999999999999999999+118"
1311                     },
1312                     "/tmp": {
1313                         "capacity": 1073741824,
1314                         "kind": "tmp"
1315                     },
1316                     "/var/spool/cwl": {
1317                         "capacity": 1073741824,
1318                         "kind": "tmp"
1319                     },
1320                     "/var/spool/cwl/cwl.input.yml": {
1321                         "kind": "collection",
1322                         "path": "cwl.input.yml",
1323                         "portable_data_hash": "99999999999999999999999999999996+99"
1324                     },
1325                     "/var/spool/cwl/workflow.cwl": {
1326                         "kind": "collection",
1327                         "path": "workflow.cwl",
1328                         "portable_data_hash": "99999999999999999999999999999996+99"
1329                     },
1330                     "stdout": {
1331                         "kind": "file",
1332                         "path": "/var/spool/cwl/cwl.output.json"
1333                     }
1334                 },
1335                 "name": "scatterstep",
1336                 "output_name": "Output for step scatterstep",
1337                 "output_path": "/var/spool/cwl",
1338                 "output_ttl": 0,
1339                 "priority": 500,
1340                 "properties": {},
1341                 "runtime_constraints": {
1342                     "ram": 1073741824,
1343                     "vcpus": 1
1344                 },
1345                 "scheduling_parameters": {},
1346                 "secret_mounts": {},
1347                 "state": "Committed",
1348                 "use_existing": True,
1349                 'output_storage_classes': ["default"]
1350             }))
1351         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
1352         mockc.open().__enter__().write.assert_has_calls([mock.call(
1353 '''{
1354   "fileblub": {
1355     "basename": "token.txt",
1356     "class": "File",
1357     "location": "/keep/99999999999999999999999999999999+118/token.txt",
1358     "size": 0
1359   },
1360   "sleeptime": 5
1361 }''')])
1362
1363     # The test passes no builder.resources
1364     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
1365     @mock.patch("arvados.collection.CollectionReader")
1366     @mock.patch("arvados.collection.Collection")
1367     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
1368     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
1369         arvados_cwl.add_arv_hints()
1370
1371         api = mock.MagicMock()
1372         api._rootDesc = get_rootDesc()
1373
1374         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1375         self.assertEqual(runner.work_api, 'containers')
1376
1377         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
1378         runner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1379                                                                "portable_data_hash": "99999999999999999999999999999993+99"}
1380         runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
1381                                                                            "portable_data_hash": "99999999999999999999999999999993+99"}]}
1382
1383         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
1384         runner.ignore_docker_for_reuse = False
1385         runner.num_retries = 0
1386         runner.secret_store = cwltool.secrets.SecretStore()
1387
1388         loadingContext, runtimeContext = self.helper(runner)
1389         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
1390         loadingContext.do_update = True
1391         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
1392
1393         mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
1394
1395         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
1396         arvtool.formatgraph = None
1397         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
1398
1399         next(it).run(runtimeContext)
1400         next(it).run(runtimeContext)
1401
1402         with open("tests/wf/echo-subwf.cwl") as f:
1403             subwf = StripYAMLComments(f.read())
1404
1405         runner.api.container_requests().create.assert_called_with(
1406             body=JsonDiffMatcher({
1407                 'output_ttl': 0,
1408                 'environment': {'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp'},
1409                 'scheduling_parameters': {},
1410                 'name': u'echo-subwf',
1411                 'secret_mounts': {},
1412                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
1413                 'properties': {},
1414                 'priority': 500,
1415                 'mounts': {
1416                     '/var/spool/cwl/cwl.input.yml': {
1417                         'portable_data_hash': '99999999999999999999999999999996+99',
1418                         'kind': 'collection',
1419                         'path': 'cwl.input.yml'
1420                     },
1421                     '/var/spool/cwl/workflow.cwl': {
1422                         'portable_data_hash': '99999999999999999999999999999996+99',
1423                         'kind': 'collection',
1424                         'path': 'workflow.cwl'
1425                     },
1426                     'stdout': {
1427                         'path': '/var/spool/cwl/cwl.output.json',
1428                         'kind': 'file'
1429                     },
1430                     '/tmp': {
1431                         'kind': 'tmp',
1432                         'capacity': 1073741824
1433                     }, '/var/spool/cwl': {
1434                         'kind': 'tmp',
1435                         'capacity': 3221225472
1436                     }
1437                 },
1438                 'state': 'Committed',
1439                 'output_path': '/var/spool/cwl',
1440                 'container_image': '99999999999999999999999999999993+99',
1441                 'command': [
1442                     u'cwltool',
1443                     u'--no-container',
1444                     u'--move-outputs',
1445                     u'--preserve-entire-environment',
1446                     u'workflow.cwl',
1447                     u'cwl.input.yml'
1448                 ],
1449                 'use_existing': True,
1450                 'output_name': u'Output for step echo-subwf',
1451                 'cwd': '/var/spool/cwl',
1452                 'output_storage_classes': ["default"]
1453             }))
1454
1455     def test_default_work_api(self):
1456         arvados_cwl.add_arv_hints()
1457
1458         api = mock.MagicMock()
1459         api._rootDesc = copy.deepcopy(get_rootDesc())
1460         runner = arvados_cwl.executor.ArvCwlExecutor(api)
1461         self.assertEqual(runner.work_api, 'containers')