sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import itertools
  14 import functools
  15 import hashlib
  16 import json
  17 import logging
  18 import mock
  19 import sys
  20 import unittest
  21 import cwltool.process
  22 import re
  23 import os
  24
  25 from io import BytesIO
  26
  27 # StringIO.StringIO and io.StringIO have different behavior write() is
  28 # called with both python2 (byte) strings and unicode strings
  29 # (specifically there's some logging in cwltool that causes trouble).
  30 # This isn't a problem on python3 because all string are unicode.
  31 if sys.version_info[0] < 3:
  32     from StringIO import StringIO
  33 else:
  34     from io import StringIO
  35
  36 import arvados
  37 import arvados.collection
  38 import arvados_cwl
  39 import arvados_cwl.executor
  40 import arvados_cwl.runner
  41 import arvados.keep
  42
  43 from .matcher import JsonDiffMatcher, StripYAMLComments
  44 from .mock_discovery import get_rootDesc
  45
  46 import ruamel.yaml
  47
  48 _rootDesc = None
  49
  50 def stubs(wfdetails=('submit_wf.cwl', None)):
  51     def outer_wrapper(func, *rest):
  52         @functools.wraps(func)
  53         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  54         @mock.patch("uuid.uuid4")
  55         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  56         @mock.patch("arvados.collection.KeepClient")
  57         @mock.patch("arvados.keep.KeepClient")
  58         @mock.patch("arvados.events.subscribe")
  59         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  60                     uuid4, determine_image_id, *args, **kwargs):
  61             class Stubs(object):
  62                 pass
  63
  64             wfname = wfdetails[0]
  65             wfpath = wfdetails[1]
  66
  67             stubs = Stubs()
  68             stubs.events = events
  69             stubs.keepdocker = keepdocker
  70
  71             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  73                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  74                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  75                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  76                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  77                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  78                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  79                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  80
  81             determine_image_id.return_value = None
  82
  83             def putstub(p, **kwargs):
  84                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  85             keep_client1().put.side_effect = putstub
  86             keep_client1.put.side_effect = putstub
  87             keep_client2().put.side_effect = putstub
  88             keep_client2.put.side_effect = putstub
  89
  90             stubs.keep_client = keep_client2
  91             stubs.docker_images = {
  92                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  93                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  94                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  95                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  96             }
  97             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  98                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  99             stubs.keepdocker.side_effect = kd
 100
 101             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
 102             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
 103
 104             if sys.version_info[0] < 3:
 105                 stubs.capture_stdout = BytesIO()
 106             else:
 107                 stubs.capture_stdout = StringIO()
 108
 109             stubs.api = mock.MagicMock()
 110             stubs.api._rootDesc = get_rootDesc()
 111             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 112             stubs.api._rootDesc["revision"] = "20210628"
 113
 114             stubs.api.users().current().execute.return_value = {
 115                 "uuid": stubs.fake_user_uuid,
 116             }
 117             stubs.api.collections().list().execute.return_value = {"items": []}
 118             stubs.api.containers().current().execute.return_value = {
 119                 "uuid": stubs.fake_container_uuid,
 120             }
 121             stubs.api.config()["StorageClasses"].items.return_value = {
 122                 "default": {
 123                     "Default": True
 124                 }
 125             }.items()
 126
 127             class CollectionExecute(object):
 128                 def __init__(self, exe):
 129                     self.exe = exe
 130                 def execute(self, num_retries=None):
 131                     return self.exe
 132
 133             def collection_createstub(created_collections, body, ensure_unique_name=None):
 134                 mt = body["manifest_text"].encode('utf-8')
 135                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 136                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 137                 created_collections[uuid] = {
 138                     "uuid": uuid,
 139                     "portable_data_hash": pdh,
 140                     "manifest_text": mt.decode('utf-8')
 141                 }
 142                 return CollectionExecute(created_collections[uuid])
 143
 144             def collection_getstub(created_collections, uuid):
 145                 for v in viewvalues(created_collections):
 146                     if uuid in (v["uuid"], v["portable_data_hash"]):
 147                         return CollectionExecute(v)
 148
 149             created_collections = {
 150                 "99999999999999999999999999999998+99": {
 151                     "uuid": "",
 152                     "portable_data_hash": "99999999999999999999999999999998+99",
 153                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 154                 },
 155                 "99999999999999999999999999999997+99": {
 156                     "uuid": "",
 157                     "portable_data_hash": "99999999999999999999999999999997+99",
 158                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 159                 },
 160                 "99999999999999999999999999999994+99": {
 161                     "uuid": "",
 162                     "portable_data_hash": "99999999999999999999999999999994+99",
 163                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 164                 },
 165                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 166                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 167                     "portable_data_hash": "999999999999999999999999999999d3+99",
 168                     "manifest_text": ""
 169                 },
 170                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 171                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 172                     "portable_data_hash": "999999999999999999999999999999d4+99",
 173                     "manifest_text": ""
 174                 },
 175                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 176                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 177                     "portable_data_hash": "999999999999999999999999999999d5+99",
 178                     "manifest_text": ""
 179                 },
 180                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 181                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 182                     "portable_data_hash": "999999999999999999999999999999d6+99",
 183                     "manifest_text": ""
 184                 }
 185             }
 186             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 187             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 188
 189             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 190             stubs.api.container_requests().create().execute.return_value = {
 191                 "uuid": stubs.expect_container_request_uuid,
 192                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 193                 "state": "Queued"
 194             }
 195
 196             cwd = os.getcwd()
 197             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 198             with open(filepath) as f:
 199                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 200                 expect_packed_workflow = yaml.load(f)
 201
 202             if wfpath is None:
 203                 wfpath = wfname
 204
 205             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 206             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 207             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 208
 209             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 210             expect_packed_workflow.update(stubs.git_info)
 211
 212             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 213
 214             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 215             if wfname == wfpath:
 216                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 217             else:
 218                 container_name = wfname
 219
 220             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 221             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 222
 223             stubs.expect_container_spec = {
 224                 'priority': 500,
 225                 'mounts': {
 226                     '/var/spool/cwl': {
 227                         'writable': True,
 228                         'kind': 'collection'
 229                     },
 230                     '/var/lib/cwl/workflow.json': {
 231                         'content': expect_packed_workflow,
 232                         'kind': 'json'
 233                     },
 234                     'stdout': {
 235                         'path': '/var/spool/cwl/cwl.output.json',
 236                         'kind': 'file'
 237                     },
 238                     '/var/lib/cwl/cwl.input.json': {
 239                         'kind': 'json',
 240                         'content': {
 241                             'y': {
 242                                 'basename': '99999999999999999999999999999998+99',
 243                                 'location': 'keep:99999999999999999999999999999998+99',
 244                                 'class': 'Directory'},
 245                             'x': {
 246                                 'basename': u'blorp.txt',
 247                                 'class': 'File',
 248                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 249                                 "size": 16
 250                             },
 251                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 252                                 {'basename': 'renamed.txt',
 253                                  'class': 'File',
 254                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 255                                  'size': 0
 256                                 }
 257                             ]}
 258                         },
 259                         'kind': 'json'
 260                     }
 261                 },
 262                 'secret_mounts': {},
 263                 'state': 'Committed',
 264                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 265                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 266                             '--eval-timeout=20', '--thread-count=0',
 267                             '--enable-reuse', "--collection-cache-size=256",
 268                             '--output-name=Output from workflow '+container_name,
 269                             '--debug', '--on-error=continue',
 270                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 271                 'name': container_name,
 272                 'container_image': '999999999999999999999999999999d3+99',
 273                 'output_name': 'Output from workflow %s' % (container_name),
 274                 'output_path': '/var/spool/cwl',
 275                 'cwd': '/var/spool/cwl',
 276                 'runtime_constraints': {
 277                     'API': True,
 278                     'vcpus': 1,
 279                     'ram': (1024+256)*1024*1024
 280                 },
 281                 'properties': stubs.git_props,
 282                 'use_existing': False,
 283                 'secret_mounts': {}
 284             }
 285
 286             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 287             stubs.api.workflows().create().execute.return_value = {
 288                 "uuid": stubs.expect_workflow_uuid,
 289             }
 290             def update_mock(**kwargs):
 291                 stubs.updated_uuid = kwargs.get('uuid')
 292                 return mock.DEFAULT
 293             stubs.api.workflows().update.side_effect = update_mock
 294             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 295                 "uuid": stubs.updated_uuid,
 296             }
 297
 298             return func(self, stubs, *args, **kwargs)
 299         return wrapped
 300     return outer_wrapper
 301
 302 class TestSubmit(unittest.TestCase):
 303
 304     def setUp(self):
 305         cwltool.process._names = set()
 306         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 307
 308     def tearDown(self):
 309         root_logger = logging.getLogger('')
 310
 311         # Remove existing RuntimeStatusLoggingHandlers if they exist
 312         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 313         root_logger.handlers = handlers
 314
 315     @mock.patch("time.sleep")
 316     @stubs()
 317     def test_submit_invalid_runner_ram(self, stubs, tm):
 318         exited = arvados_cwl.main(
 319             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 320              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 321             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 322         self.assertEqual(exited, 1)
 323
 324
 325     @stubs()
 326     def test_submit_container(self, stubs):
 327         exited = arvados_cwl.main(
 328             ["--submit", "--no-wait", "--api=containers", "--debug",
 329                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 330             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 331
 332         stubs.api.collections().create.assert_has_calls([
 333             mock.call(body=JsonDiffMatcher({
 334                 'manifest_text':
 335                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 336                 'replication_desired': None,
 337                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 338             }), ensure_unique_name=False),
 339             mock.call(body=JsonDiffMatcher({
 340                 'manifest_text':
 341                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 342                 'replication_desired': None,
 343                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 344             }), ensure_unique_name=False),
 345             ])
 346
 347         expect_container = copy.deepcopy(stubs.expect_container_spec)
 348         stubs.api.container_requests().create.assert_called_with(
 349             body=JsonDiffMatcher(expect_container))
 350         self.assertEqual(stubs.capture_stdout.getvalue(),
 351                          stubs.expect_container_request_uuid + '\n')
 352         self.assertEqual(exited, 0)
 353
 354
 355     @stubs()
 356     def test_submit_container_tool(self, stubs):
 357         # test for issue #16139
 358         exited = arvados_cwl.main(
 359             ["--submit", "--no-wait", "--api=containers", "--debug",
 360                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 361             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 362
 363         self.assertEqual(stubs.capture_stdout.getvalue(),
 364                          stubs.expect_container_request_uuid + '\n')
 365         self.assertEqual(exited, 0)
 366
 367     @stubs()
 368     def test_submit_container_no_reuse(self, stubs):
 369         exited = arvados_cwl.main(
 370             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 371                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 372             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 373
 374         expect_container = copy.deepcopy(stubs.expect_container_spec)
 375         expect_container["command"] = [
 376             'arvados-cwl-runner', '--local', '--api=containers',
 377             '--no-log-timestamps', '--disable-validate', '--disable-color',
 378             '--eval-timeout=20', '--thread-count=0',
 379             '--disable-reuse', "--collection-cache-size=256",
 380             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 381             '--debug', '--on-error=continue',
 382             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 383         expect_container["use_existing"] = False
 384
 385         stubs.api.container_requests().create.assert_called_with(
 386             body=JsonDiffMatcher(expect_container))
 387         self.assertEqual(stubs.capture_stdout.getvalue(),
 388                          stubs.expect_container_request_uuid + '\n')
 389         self.assertEqual(exited, 0)
 390
 391     @stubs(('submit_wf_no_reuse.cwl', None))
 392     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 393         exited = arvados_cwl.main(
 394             ["--submit", "--no-wait", "--api=containers", "--debug",
 395              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 396             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 397         self.assertEqual(exited, 0)
 398
 399         expect_container = copy.deepcopy(stubs.expect_container_spec)
 400         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 401         expect_container["use_existing"] = False
 402         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 403             {
 404                 "class": "WorkReuse",
 405                 "enableReuse": False,
 406             },
 407                 {
 408                     "acrContainerImage": "999999999999999999999999999999d3+99",
 409                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 410                 }
 411         ]
 412         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 413
 414         stubs.api.container_requests().create.assert_called_with(
 415             body=JsonDiffMatcher(expect_container))
 416         self.assertEqual(stubs.capture_stdout.getvalue(),
 417                          stubs.expect_container_request_uuid + '\n')
 418
 419
 420     @stubs()
 421     def test_submit_container_on_error(self, stubs):
 422         exited = arvados_cwl.main(
 423             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 424                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 426
 427         expect_container = copy.deepcopy(stubs.expect_container_spec)
 428         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 429                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 430                                        '--eval-timeout=20', '--thread-count=0',
 431                                        '--enable-reuse', "--collection-cache-size=256",
 432                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 433                                        '--debug', '--on-error=stop',
 434                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 435
 436         stubs.api.container_requests().create.assert_called_with(
 437             body=JsonDiffMatcher(expect_container))
 438         self.assertEqual(stubs.capture_stdout.getvalue(),
 439                          stubs.expect_container_request_uuid + '\n')
 440         self.assertEqual(exited, 0)
 441
 442     @stubs()
 443     def test_submit_container_output_name(self, stubs):
 444         output_name = "test_output_name"
 445
 446         exited = arvados_cwl.main(
 447             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 448                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 449             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 450
 451         expect_container = copy.deepcopy(stubs.expect_container_spec)
 452         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 453                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 454                                        '--eval-timeout=20', '--thread-count=0',
 455                                        '--enable-reuse', "--collection-cache-size=256",
 456                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 457                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 458         expect_container["output_name"] = output_name
 459
 460         stubs.api.container_requests().create.assert_called_with(
 461             body=JsonDiffMatcher(expect_container))
 462         self.assertEqual(stubs.capture_stdout.getvalue(),
 463                          stubs.expect_container_request_uuid + '\n')
 464         self.assertEqual(exited, 0)
 465
 466     @stubs()
 467     def test_submit_storage_classes(self, stubs):
 468         exited = arvados_cwl.main(
 469             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 470                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 471             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 472
 473         expect_container = copy.deepcopy(stubs.expect_container_spec)
 474         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 475                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 476                                        '--eval-timeout=20', '--thread-count=0',
 477                                        '--enable-reuse', "--collection-cache-size=256",
 478                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 479                                        "--debug",
 480                                        "--storage-classes=foo", '--on-error=continue',
 481                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 482
 483         stubs.api.container_requests().create.assert_called_with(
 484             body=JsonDiffMatcher(expect_container))
 485         self.assertEqual(stubs.capture_stdout.getvalue(),
 486                          stubs.expect_container_request_uuid + '\n')
 487         self.assertEqual(exited, 0)
 488
 489     @stubs()
 490     def test_submit_multiple_storage_classes(self, stubs):
 491         exited = arvados_cwl.main(
 492             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 493                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 494             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 495
 496         expect_container = copy.deepcopy(stubs.expect_container_spec)
 497         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 498                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 499                                        '--eval-timeout=20', '--thread-count=0',
 500                                        '--enable-reuse', "--collection-cache-size=256",
 501                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 502                                        "--debug",
 503                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 504                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(stubs.capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510         self.assertEqual(exited, 0)
 511
 512     @mock.patch("cwltool.task_queue.TaskQueue")
 513     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 514     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 515     @stubs()
 516     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 517         final_output_c = arvados.collection.Collection()
 518         make_output.return_value = ({},final_output_c)
 519
 520         def set_final_output(job_order, output_callback, runtimeContext):
 521             output_callback({"out": "zzzzz"}, "success")
 522             return []
 523         job.side_effect = set_final_output
 524
 525         exited = arvados_cwl.main(
 526             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 527                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 528             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 529
 530         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 531         self.assertEqual(exited, 0)
 532
 533     @mock.patch("cwltool.task_queue.TaskQueue")
 534     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 535     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 536     @stubs()
 537     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 538         final_output_c = arvados.collection.Collection()
 539         make_output.return_value = ({},final_output_c)
 540         stubs.api.config().get.return_value = {"default": {"Default": True}}
 541
 542         def set_final_output(job_order, output_callback, runtimeContext):
 543             output_callback({"out": "zzzzz"}, "success")
 544             return []
 545         job.side_effect = set_final_output
 546
 547         exited = arvados_cwl.main(
 548             ["--debug", "--local", "--disable-git",
 549                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 550             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 551
 552         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 553         self.assertEqual(exited, 0)
 554
 555     @mock.patch("cwltool.task_queue.TaskQueue")
 556     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 557     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 558     @stubs()
 559     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 560         final_output_c = arvados.collection.Collection()
 561         make_output.return_value = ({},final_output_c)
 562
 563         def set_final_output(job_order, output_callback, runtimeContext):
 564             output_callback({"out": "zzzzz"}, "success")
 565             return []
 566         job.side_effect = set_final_output
 567
 568         exited = arvados_cwl.main(
 569             ["--debug", "--local", "--disable-git",
 570                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 571             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 572
 573         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 574         self.assertEqual(exited, 0)
 575
 576     @stubs()
 577     def test_submit_container_output_ttl(self, stubs):
 578         exited = arvados_cwl.main(
 579             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 580                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 581             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 582
 583         expect_container = copy.deepcopy(stubs.expect_container_spec)
 584         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 585                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 586                                        '--eval-timeout=20', '--thread-count=0',
 587                                        '--enable-reuse', "--collection-cache-size=256",
 588                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 589                                        '--debug',
 590                                        '--on-error=continue',
 591                                        "--intermediate-output-ttl=3600",
 592                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 593
 594         stubs.api.container_requests().create.assert_called_with(
 595             body=JsonDiffMatcher(expect_container))
 596         self.assertEqual(stubs.capture_stdout.getvalue(),
 597                          stubs.expect_container_request_uuid + '\n')
 598         self.assertEqual(exited, 0)
 599
 600     @stubs()
 601     def test_submit_container_trash_intermediate(self, stubs):
 602         exited = arvados_cwl.main(
 603             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 604                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 605             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 606
 607
 608         expect_container = copy.deepcopy(stubs.expect_container_spec)
 609         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 610                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 611                                        '--eval-timeout=20', '--thread-count=0',
 612                                        '--enable-reuse', "--collection-cache-size=256",
 613                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 614                                        '--debug', '--on-error=continue',
 615                                        "--trash-intermediate",
 616                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 617
 618         stubs.api.container_requests().create.assert_called_with(
 619             body=JsonDiffMatcher(expect_container))
 620         self.assertEqual(stubs.capture_stdout.getvalue(),
 621                          stubs.expect_container_request_uuid + '\n')
 622         self.assertEqual(exited, 0)
 623
 624     @stubs()
 625     def test_submit_container_output_tags(self, stubs):
 626         output_tags = "tag0,tag1,tag2"
 627
 628         exited = arvados_cwl.main(
 629             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 630                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 631             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 632
 633         expect_container = copy.deepcopy(stubs.expect_container_spec)
 634         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 635                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 636                                        '--eval-timeout=20', '--thread-count=0',
 637                                        '--enable-reuse', "--collection-cache-size=256",
 638                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 639                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 640                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 641
 642         stubs.api.container_requests().create.assert_called_with(
 643             body=JsonDiffMatcher(expect_container))
 644         self.assertEqual(stubs.capture_stdout.getvalue(),
 645                          stubs.expect_container_request_uuid + '\n')
 646         self.assertEqual(exited, 0)
 647
 648     @stubs()
 649     def test_submit_container_runner_ram(self, stubs):
 650         exited = arvados_cwl.main(
 651             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 652                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 653             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 654
 655         expect_container = copy.deepcopy(stubs.expect_container_spec)
 656         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 657         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 658                 {
 659                     "acrContainerImage": "999999999999999999999999999999d3+99",
 660                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 661                     "ramMin": 2048
 662                 }
 663         ]
 664
 665         stubs.api.container_requests().create.assert_called_with(
 666             body=JsonDiffMatcher(expect_container))
 667         self.assertEqual(stubs.capture_stdout.getvalue(),
 668                          stubs.expect_container_request_uuid + '\n')
 669         self.assertEqual(exited, 0)
 670
 671     @mock.patch("arvados.collection.CollectionReader")
 672     @mock.patch("time.sleep")
 673     @stubs()
 674     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 675         collectionReader().exists.return_value = True
 676         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 677         exited = arvados_cwl.main(
 678             ["--submit", "--no-wait", "--api=containers", "--debug",
 679              "tests/wf/submit_keepref_wf.cwl"],
 680             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 681         self.assertEqual(exited, 0)
 682
 683     @mock.patch("arvados.collection.CollectionReader")
 684     @mock.patch("time.sleep")
 685     @stubs()
 686     def test_submit_keepref(self, stubs, tm, reader):
 687         with open("tests/wf/expect_arvworkflow.cwl") as f:
 688             reader().open().__enter__().read.return_value = f.read()
 689
 690         exited = arvados_cwl.main(
 691             ["--submit", "--no-wait", "--api=containers", "--debug",
 692              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 693             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 694
 695         expect_container = {
 696             'priority': 500,
 697             'mounts': {
 698                 '/var/spool/cwl': {
 699                     'writable': True,
 700                     'kind': 'collection'
 701                 },
 702                 'stdout': {
 703                     'path': '/var/spool/cwl/cwl.output.json',
 704                     'kind': 'file'
 705                 },
 706                 '/var/lib/cwl/workflow': {
 707                     'portable_data_hash': '99999999999999999999999999999994+99',
 708                     'kind': 'collection'
 709                 },
 710                 '/var/lib/cwl/cwl.input.json': {
 711                     'content': {
 712                         'x': 'XxX'
 713                     },
 714                     'kind': 'json'
 715                 }
 716             }, 'state': 'Committed',
 717             'output_path': '/var/spool/cwl',
 718             'name': 'expect_arvworkflow.cwl#main',
 719             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 720             'container_image': '999999999999999999999999999999d3+99',
 721             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 722                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 723                         '--eval-timeout=20', '--thread-count=0',
 724                         '--enable-reuse', "--collection-cache-size=256",
 725                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 726                         '--debug', '--on-error=continue',
 727                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 728             'cwd': '/var/spool/cwl',
 729             'runtime_constraints': {
 730                 'API': True,
 731                 'vcpus': 1,
 732                 'ram': 1342177280
 733             },
 734             'use_existing': False,
 735             'properties': {},
 736             'secret_mounts': {}
 737         }
 738
 739         stubs.api.container_requests().create.assert_called_with(
 740             body=JsonDiffMatcher(expect_container))
 741         self.assertEqual(stubs.capture_stdout.getvalue(),
 742                          stubs.expect_container_request_uuid + '\n')
 743         self.assertEqual(exited, 0)
 744
 745     @mock.patch("time.sleep")
 746     @stubs()
 747     def test_submit_arvworkflow(self, stubs, tm):
 748         with open("tests/wf/expect_arvworkflow.cwl") as f:
 749             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 750
 751         exited = arvados_cwl.main(
 752             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 753              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 754             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 755
 756         expect_container = {
 757             'priority': 500,
 758             'mounts': {
 759                 '/var/spool/cwl': {
 760                     'writable': True,
 761                     'kind': 'collection'
 762                 },
 763                 'stdout': {
 764                     'path': '/var/spool/cwl/cwl.output.json',
 765                     'kind': 'file'
 766                 },
 767                 '/var/lib/cwl/workflow.json': {
 768                     'kind': 'json',
 769                     'content': {
 770                         'cwlVersion': 'v1.0',
 771                         'label': 'a test workflow',
 772                         '$graph': [
 773                             {
 774                                 'id': '#main',
 775                                 'inputs': [
 776                                     {'type': 'string', 'id': '#main/x'}
 777                                 ],
 778                                 'steps': [
 779                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 780                                      'run': '#submit_tool.cwl',
 781                                      'id': '#main/step1',
 782                                      'out': []}
 783                                 ],
 784                                 'class': 'Workflow',
 785                                 'outputs': []
 786                             },
 787                             {
 788                                 'inputs': [
 789                                     {
 790                                         'inputBinding': {'position': 1},
 791                                         'type': 'string',
 792                                         'id': '#submit_tool.cwl/x'}
 793                                 ],
 794                                 'requirements': [
 795                                     {
 796                                         'dockerPull': 'debian:buster-slim',
 797                                         'class': 'DockerRequirement'
 798                                     }
 799                                 ],
 800                                 'id': '#submit_tool.cwl',
 801                                 'outputs': [],
 802                                 'baseCommand': 'cat',
 803                                 'class': 'CommandLineTool'
 804                             }
 805                         ]
 806                     }
 807                 },
 808                 '/var/lib/cwl/cwl.input.json': {
 809                     'content': {
 810                         'x': 'XxX'
 811                     },
 812                     'kind': 'json'
 813                 }
 814             }, 'state': 'Committed',
 815             'output_path': '/var/spool/cwl',
 816             'name': 'a test workflow',
 817             'container_image': "999999999999999999999999999999d3+99",
 818             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 819                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 820                         '--eval-timeout=20', '--thread-count=0',
 821                         '--enable-reuse', "--collection-cache-size=256",
 822                         "--output-name=Output from workflow a test workflow",
 823                         '--debug', '--on-error=continue',
 824                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 825             'output_name': 'Output from workflow a test workflow',
 826             'cwd': '/var/spool/cwl',
 827             'runtime_constraints': {
 828                 'API': True,
 829                 'vcpus': 1,
 830                 'ram': 1342177280
 831             },
 832             'use_existing': False,
 833             'properties': {
 834                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 835             },
 836             'secret_mounts': {}
 837         }
 838
 839         stubs.api.container_requests().create.assert_called_with(
 840             body=JsonDiffMatcher(expect_container))
 841         self.assertEqual(stubs.capture_stdout.getvalue(),
 842                          stubs.expect_container_request_uuid + '\n')
 843         self.assertEqual(exited, 0)
 844
 845     @stubs(('hello container 123', 'submit_wf.cwl'))
 846     def test_submit_container_name(self, stubs):
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 849                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851
 852         expect_container = copy.deepcopy(stubs.expect_container_spec)
 853
 854         stubs.api.container_requests().create.assert_called_with(
 855             body=JsonDiffMatcher(expect_container))
 856         self.assertEqual(stubs.capture_stdout.getvalue(),
 857                          stubs.expect_container_request_uuid + '\n')
 858         self.assertEqual(exited, 0)
 859
 860     @stubs()
 861     def test_submit_missing_input(self, stubs):
 862         exited = arvados_cwl.main(
 863             ["--submit", "--no-wait", "--api=containers", "--debug",
 864              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 865             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 866         self.assertEqual(exited, 0)
 867
 868         exited = arvados_cwl.main(
 869             ["--submit", "--no-wait", "--api=containers", "--debug",
 870              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 871             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 872         self.assertEqual(exited, 1)
 873
 874     @stubs()
 875     def test_submit_container_project(self, stubs):
 876         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 877         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 878         exited = arvados_cwl.main(
 879             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 880                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 881             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 882
 883         expect_container = copy.deepcopy(stubs.expect_container_spec)
 884         expect_container["owner_uuid"] = project_uuid
 885         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 886                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 887                                        "--eval-timeout=20", "--thread-count=0",
 888                                        '--enable-reuse', "--collection-cache-size=256",
 889                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 890                                        '--debug',
 891                                        '--on-error=continue',
 892                                        '--project-uuid='+project_uuid,
 893                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 894
 895         stubs.api.container_requests().create.assert_called_with(
 896             body=JsonDiffMatcher(expect_container))
 897         self.assertEqual(stubs.capture_stdout.getvalue(),
 898                          stubs.expect_container_request_uuid + '\n')
 899         self.assertEqual(exited, 0)
 900
 901     @stubs()
 902     def test_submit_container_eval_timeout(self, stubs):
 903         exited = arvados_cwl.main(
 904             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 905                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 906             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 907
 908         expect_container = copy.deepcopy(stubs.expect_container_spec)
 909         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 910                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 911                                        '--eval-timeout=60.0', '--thread-count=0',
 912                                        '--enable-reuse', "--collection-cache-size=256",
 913                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 914                                        '--debug', '--on-error=continue',
 915                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 916
 917         stubs.api.container_requests().create.assert_called_with(
 918             body=JsonDiffMatcher(expect_container))
 919         self.assertEqual(stubs.capture_stdout.getvalue(),
 920                          stubs.expect_container_request_uuid + '\n')
 921         self.assertEqual(exited, 0)
 922
 923     @stubs()
 924     def test_submit_container_collection_cache(self, stubs):
 925         exited = arvados_cwl.main(
 926             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 927                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 928             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 929
 930         expect_container = copy.deepcopy(stubs.expect_container_spec)
 931         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 932                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 933                                        '--eval-timeout=20', '--thread-count=0',
 934                                        '--enable-reuse', "--collection-cache-size=500",
 935                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 936                                        '--debug', '--on-error=continue',
 937                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 938         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 939
 940         stubs.api.container_requests().create.assert_called_with(
 941             body=JsonDiffMatcher(expect_container))
 942         self.assertEqual(stubs.capture_stdout.getvalue(),
 943                          stubs.expect_container_request_uuid + '\n')
 944         self.assertEqual(exited, 0)
 945
 946     @stubs()
 947     def test_submit_container_thread_count(self, stubs):
 948         exited = arvados_cwl.main(
 949             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 950                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 951             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 952
 953         expect_container = copy.deepcopy(stubs.expect_container_spec)
 954         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 955                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 956                                        '--eval-timeout=20', '--thread-count=20',
 957                                        '--enable-reuse', "--collection-cache-size=256",
 958                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 959                                        '--debug', '--on-error=continue',
 960                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 961
 962         stubs.api.container_requests().create.assert_called_with(
 963             body=JsonDiffMatcher(expect_container))
 964         self.assertEqual(stubs.capture_stdout.getvalue(),
 965                          stubs.expect_container_request_uuid + '\n')
 966         self.assertEqual(exited, 0)
 967
 968     @stubs()
 969     def test_submit_container_runner_image(self, stubs):
 970         exited = arvados_cwl.main(
 971             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 972                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 973             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 974
 975         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 976         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 977                 {
 978                     "acrContainerImage": "999999999999999999999999999999d5+99",
 979                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 980                 }
 981         ]
 982
 983         expect_container = copy.deepcopy(stubs.expect_container_spec)
 984         stubs.api.container_requests().create.assert_called_with(
 985             body=JsonDiffMatcher(expect_container))
 986         self.assertEqual(stubs.capture_stdout.getvalue(),
 987                          stubs.expect_container_request_uuid + '\n')
 988         self.assertEqual(exited, 0)
 989
 990     @stubs()
 991     def test_submit_priority(self, stubs):
 992         exited = arvados_cwl.main(
 993             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 994                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 995             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 996
 997         stubs.expect_container_spec["priority"] = 669
 998
 999         expect_container = copy.deepcopy(stubs.expect_container_spec)
1000         stubs.api.container_requests().create.assert_called_with(
1001             body=JsonDiffMatcher(expect_container))
1002         self.assertEqual(stubs.capture_stdout.getvalue(),
1003                          stubs.expect_container_request_uuid + '\n')
1004         self.assertEqual(exited, 0)
1005
1006     @stubs(('submit_wf_runner_resources.cwl', None))
1007     def test_submit_wf_runner_resources(self, stubs):
1008         exited = arvados_cwl.main(
1009             ["--submit", "--no-wait", "--api=containers", "--debug",
1010                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1011             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1012
1013         expect_container = copy.deepcopy(stubs.expect_container_spec)
1014         expect_container["runtime_constraints"] = {
1015             "API": True,
1016             "vcpus": 2,
1017             "ram": (2000+512) * 2**20
1018         }
1019         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1020             {
1021                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1022                 "acrContainerImage": "999999999999999999999999999999d3+99",
1023                 "coresMin": 2,
1024                 "ramMin": 2000,
1025                 "keep_cache": 512
1026             }
1027         ]
1028         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1029         #    "arv": "http://arvados.org/cwl#",
1030         #}
1031         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1032         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1033
1034         stubs.api.container_requests().create.assert_called_with(
1035             body=JsonDiffMatcher(expect_container))
1036         self.assertEqual(stubs.capture_stdout.getvalue(),
1037                          stubs.expect_container_request_uuid + '\n')
1038         self.assertEqual(exited, 0)
1039
1040     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1041     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1042     @mock.patch("arvados.api")
1043     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1044         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1045
1046         arvrunner = mock.MagicMock()
1047         arvrunner.project_uuid = ""
1048         api.return_value = mock.MagicMock()
1049         arvrunner.api = api.return_value
1050         arvrunner.runtimeContext.match_local_docker = False
1051         arvrunner.api.links().list().execute.side_effect = itertools.cycle([
1052             {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
1053                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1054                         "link_class": "docker_image_repo+tag",
1055                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1056                         "owner_uuid": "",
1057                         "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
1058                         "properties": {"image_timestamp": ""}}]},
1059             {"items": []},
1060             {"items": []},
1061             {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
1062                         "head_uuid": "",
1063                         "link_class": "docker_image_hash",
1064                         "name": "123456",
1065                         "owner_uuid": "",
1066                         "uuid": "zzzzz-o0j2j-arvadosjobshash",
1067                         "properties": {"image_timestamp": ""}}]},
1068             {"items": []},
1069             {"items": []},
1070         ])
1071         find_one_image_hash.return_value = "123456"
1072
1073         arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
1074             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1075                         "owner_uuid": "",
1076                         "manifest_text": "",
1077                         "created_at": "2023-08-25T12:34:55.012345Z",
1078                         "properties": {}}]},
1079             {"items": []},
1080             {"items": []},
1081         ])
1082         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1083         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1084                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1085
1086         self.assertEqual("9999999999999999999999999999999b+99",
1087                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1088
1089
1090     @stubs()
1091     def test_submit_secrets(self, stubs):
1092         exited = arvados_cwl.main(
1093             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1094                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1095             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1096
1097         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1098         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1099
1100         expect_container = {
1101             "command": [
1102                 "arvados-cwl-runner",
1103                 "--local",
1104                 "--api=containers",
1105                 "--no-log-timestamps",
1106                 "--disable-validate",
1107                 "--disable-color",
1108                 "--eval-timeout=20",
1109                 '--thread-count=0',
1110                 "--enable-reuse",
1111                 "--collection-cache-size=256",
1112                 "--output-name=Output from workflow secret_wf.cwl",
1113                 "--debug",
1114                 "--on-error=continue",
1115                 "/var/lib/cwl/workflow.json#main",
1116                 "/var/lib/cwl/cwl.input.json"
1117             ],
1118             "container_image": "999999999999999999999999999999d3+99",
1119             "cwd": "/var/spool/cwl",
1120             "mounts": {
1121                 "/var/lib/cwl/cwl.input.json": {
1122                     "content": {
1123                         "pw": {
1124                             "$include": "/secrets/s0"
1125                         }
1126                     },
1127                     "kind": "json"
1128                 },
1129                 "/var/lib/cwl/workflow.json": {
1130                     "content": {
1131                         "$graph": [
1132                             {
1133                                 "class": "Workflow",
1134                                 "hints": [
1135                                     {
1136                                     "class": "DockerRequirement",
1137                                     "dockerPull": "debian:buster-slim",
1138                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1139                                     },
1140                                     {
1141                                         "class": "http://commonwl.org/cwltool#Secrets",
1142                                         "secrets": [
1143                                             "#main/pw"
1144                                         ]
1145                                     },
1146                                     {
1147                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1148                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1149                                     }
1150                                 ],
1151                                 "id": "#main",
1152                                 "inputs": [
1153                                     {
1154                                         "id": "#main/pw",
1155                                         "type": "string"
1156                                     }
1157                                 ],
1158                                 "outputs": [
1159                                     {
1160                                         "id": "#main/out",
1161                                         "outputSource": "#main/step/out",
1162                                         "type": "File"
1163                                     }
1164                                 ],
1165                                 "requirements": [
1166                                     {
1167                                         "class": "SubworkflowFeatureRequirement"
1168                                     }
1169                                 ],
1170                                 "steps": [
1171                                     {
1172                                         "id": "#main/secret_wf.cwl",
1173                                         "in": [
1174                                             {
1175                                                 "id": "#main/step/pw",
1176                                                 "source": "#main/pw"
1177                                             }
1178                                         ],
1179                                         "label": "secret_wf.cwl",
1180                                         "out": [
1181                                             {"id": "#main/step/out"}
1182                                         ],
1183                                         "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
1184                                     }
1185                                 ]
1186                             }
1187                         ],
1188                         "cwlVersion": "v1.2"
1189                     },
1190                     "kind": "json"
1191                 },
1192                 "/var/spool/cwl": {
1193                     "kind": "collection",
1194                     "writable": True
1195                 },
1196                 "stdout": {
1197                     "kind": "file",
1198                     "path": "/var/spool/cwl/cwl.output.json"
1199                 }
1200             },
1201             "name": "secret_wf.cwl",
1202             "output_name": "Output from workflow secret_wf.cwl",
1203             "output_path": "/var/spool/cwl",
1204             "priority": 500,
1205             "properties": {},
1206             "runtime_constraints": {
1207                 "API": True,
1208                 "ram": 1342177280,
1209                 "vcpus": 1
1210             },
1211             "secret_mounts": {
1212                 "/secrets/s0": {
1213                     "content": "blorp",
1214                     "kind": "text"
1215                 }
1216             },
1217             "state": "Committed",
1218             "use_existing": False
1219         }
1220
1221         stubs.api.container_requests().create.assert_called_with(
1222             body=JsonDiffMatcher(expect_container))
1223         self.assertEqual(stubs.capture_stdout.getvalue(),
1224                          stubs.expect_container_request_uuid + '\n')
1225         self.assertEqual(exited, 0)
1226
1227     @stubs()
1228     def test_submit_request_uuid(self, stubs):
1229         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1230         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1231
1232         stubs.api.container_requests().update().execute.return_value = {
1233             "uuid": stubs.expect_container_request_uuid,
1234             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1235             "state": "Queued"
1236         }
1237
1238         exited = arvados_cwl.main(
1239             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1240                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1241             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1242
1243         stubs.api.container_requests().update.assert_called_with(
1244             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1245         self.assertEqual(stubs.capture_stdout.getvalue(),
1246                          stubs.expect_container_request_uuid + '\n')
1247         self.assertEqual(exited, 0)
1248
1249     @stubs()
1250     def test_submit_container_cluster_id(self, stubs):
1251         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1252
1253         exited = arvados_cwl.main(
1254             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1255                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1256             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1257
1258         expect_container = copy.deepcopy(stubs.expect_container_spec)
1259
1260         stubs.api.container_requests().create.assert_called_with(
1261             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1262         self.assertEqual(stubs.capture_stdout.getvalue(),
1263                          stubs.expect_container_request_uuid + '\n')
1264         self.assertEqual(exited, 0)
1265
1266     @stubs()
1267     def test_submit_validate_cluster_id(self, stubs):
1268         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1269         exited = arvados_cwl.main(
1270             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1271              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1272             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1273         self.assertEqual(exited, 1)
1274
1275     @stubs()
1276     def test_submit_validate_project_uuid(self, stubs):
1277         # Fails with bad cluster prefix
1278         exited = arvados_cwl.main(
1279             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1281             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1282         self.assertEqual(exited, 1)
1283
1284         # Project lookup fails
1285         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1286         exited = arvados_cwl.main(
1287             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1288              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1289             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1290         self.assertEqual(exited, 1)
1291
1292         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1293         exited = arvados_cwl.main(
1294             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1295              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1296             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1297         self.assertEqual(exited, 0)
1298
1299
1300     @mock.patch("arvados.collection.CollectionReader")
1301     @stubs()
1302     def test_submit_uuid_inputs(self, stubs, collectionReader):
1303         collectionReader().exists.return_value = True
1304         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1305         def list_side_effect(**kwargs):
1306             m = mock.MagicMock()
1307             if "count" in kwargs:
1308                 m.execute.return_value = {"items": [
1309                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1310                 ]}
1311             else:
1312                 m.execute.return_value = {"items": []}
1313             return m
1314         stubs.api.collections().list.side_effect = list_side_effect
1315         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1316
1317         exited = arvados_cwl.main(
1318             ["--submit", "--no-wait", "--api=containers", "--debug",
1319                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1320             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1321
1322         expect_container = copy.deepcopy(stubs.expect_container_spec)
1323         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1324         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1325         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1326
1327         stubs.api.collections().list.assert_has_calls([
1328             mock.call(count='none',
1329                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1330                       select=['uuid', 'portable_data_hash'])])
1331         stubs.api.container_requests().create.assert_called_with(
1332             body=JsonDiffMatcher(expect_container))
1333         self.assertEqual(stubs.capture_stdout.getvalue(),
1334                          stubs.expect_container_request_uuid + '\n')
1335         self.assertEqual(exited, 0)
1336
1337     @stubs()
1338     def test_submit_mismatched_uuid_inputs(self, stubs):
1339         def list_side_effect(**kwargs):
1340             m = mock.MagicMock()
1341             if "count" in kwargs:
1342                 m.execute.return_value = {"items": [
1343                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1344                 ]}
1345             else:
1346                 m.execute.return_value = {"items": []}
1347             return m
1348         stubs.api.collections().list.side_effect = list_side_effect
1349
1350         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1351             capture_stderr = StringIO()
1352             cwltool_logger = logging.getLogger('cwltool')
1353             stderr_logger = logging.StreamHandler(capture_stderr)
1354             cwltool_logger.addHandler(stderr_logger)
1355
1356             try:
1357                 exited = arvados_cwl.main(
1358                     ["--submit", "--no-wait", "--api=containers", "--debug",
1359                         "tests/wf/submit_wf.cwl", infile],
1360                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1361
1362                 self.assertEqual(exited, 1)
1363                 self.assertRegex(
1364                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1365                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1366             finally:
1367                 cwltool_logger.removeHandler(stderr_logger)
1368
1369     @mock.patch("arvados.collection.CollectionReader")
1370     @stubs()
1371     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1372         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1373         capture_stderr = StringIO()
1374
1375         cwltool_logger = logging.getLogger('cwltool')
1376         stderr_logger = logging.StreamHandler(capture_stderr)
1377         cwltool_logger.addHandler(stderr_logger)
1378
1379         exited = arvados_cwl.main(
1380             ["--submit", "--no-wait", "--api=containers", "--debug",
1381                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1382             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1383
1384         try:
1385             self.assertEqual(exited, 1)
1386             self.assertRegex(
1387                 capture_stderr.getvalue(),
1388                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1389         finally:
1390             cwltool_logger.removeHandler(stderr_logger)
1391
1392     @stubs(('submit_wf_process_properties.cwl', None))
1393     def test_submit_set_process_properties(self, stubs):
1394         exited = arvados_cwl.main(
1395             ["--submit", "--no-wait", "--api=containers", "--debug",
1396                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1397             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1398
1399         expect_container = copy.deepcopy(stubs.expect_container_spec)
1400
1401         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1402             {
1403                 "class": "http://arvados.org/cwl#ProcessProperties",
1404                 "processProperties": [
1405                     {"propertyName": "baz",
1406                      "propertyValue": "$(inputs.x.basename)"},
1407                     {"propertyName": "foo",
1408                      "propertyValue": "bar"},
1409                     {"propertyName": "quux",
1410                      "propertyValue": {
1411                          "q1": 1,
1412                          "q2": 2
1413                      }
1414                     }
1415                 ],
1416             },
1417             {
1418                 "acrContainerImage": "999999999999999999999999999999d3+99",
1419                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1420             }
1421         ]
1422         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1423         #    "arv": "http://arvados.org/cwl#"
1424         #}
1425
1426         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1427
1428         expect_container["properties"].update({
1429             "baz": "blorp.txt",
1430             "foo": "bar",
1431             "quux": {
1432                 "q1": 1,
1433                 "q2": 2
1434             }
1435         })
1436
1437         stubs.api.container_requests().create.assert_called_with(
1438             body=JsonDiffMatcher(expect_container))
1439         self.assertEqual(stubs.capture_stdout.getvalue(),
1440                          stubs.expect_container_request_uuid + '\n')
1441         self.assertEqual(exited, 0)
1442
1443
1444     @stubs()
1445     def test_submit_enable_preemptible(self, stubs):
1446         exited = arvados_cwl.main(
1447             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1448                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1449             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1450
1451         expect_container = copy.deepcopy(stubs.expect_container_spec)
1452         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1453                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1454                                        '--eval-timeout=20', '--thread-count=0',
1455                                        '--enable-reuse', "--collection-cache-size=256",
1456                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1457                                        '--debug', '--on-error=continue',
1458                                        '--enable-preemptible',
1459                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1460
1461         stubs.api.container_requests().create.assert_called_with(
1462             body=JsonDiffMatcher(expect_container))
1463         self.assertEqual(stubs.capture_stdout.getvalue(),
1464                          stubs.expect_container_request_uuid + '\n')
1465         self.assertEqual(exited, 0)
1466
1467     @stubs()
1468     def test_submit_disable_preemptible(self, stubs):
1469         exited = arvados_cwl.main(
1470             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1471                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1472             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1473
1474         expect_container = copy.deepcopy(stubs.expect_container_spec)
1475         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1476                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1477                                        '--eval-timeout=20', '--thread-count=0',
1478                                        '--enable-reuse', "--collection-cache-size=256",
1479                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1480                                        '--debug', '--on-error=continue',
1481                                        '--disable-preemptible',
1482                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1483
1484         stubs.api.container_requests().create.assert_called_with(
1485             body=JsonDiffMatcher(expect_container))
1486         self.assertEqual(stubs.capture_stdout.getvalue(),
1487                          stubs.expect_container_request_uuid + '\n')
1488         self.assertEqual(exited, 0)
1489
1490     @stubs()
1491     def test_submit_container_prefer_cached_downloads(self, stubs):
1492         exited = arvados_cwl.main(
1493             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1494                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1495             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1496
1497         expect_container = copy.deepcopy(stubs.expect_container_spec)
1498         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1499                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1500                                        '--eval-timeout=20', '--thread-count=0',
1501                                        '--enable-reuse', "--collection-cache-size=256",
1502                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1503                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1504                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1505
1506         stubs.api.container_requests().create.assert_called_with(
1507             body=JsonDiffMatcher(expect_container))
1508         self.assertEqual(stubs.capture_stdout.getvalue(),
1509                          stubs.expect_container_request_uuid + '\n')
1510         self.assertEqual(exited, 0)
1511
1512     @stubs()
1513     def test_submit_container_varying_url_params(self, stubs):
1514         exited = arvados_cwl.main(
1515             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1516                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1517             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1518
1519         expect_container = copy.deepcopy(stubs.expect_container_spec)
1520         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1521                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1522                                        '--eval-timeout=20', '--thread-count=0',
1523                                        '--enable-reuse', "--collection-cache-size=256",
1524                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1525                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1526                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1527
1528         stubs.api.container_requests().create.assert_called_with(
1529             body=JsonDiffMatcher(expect_container))
1530         self.assertEqual(stubs.capture_stdout.getvalue(),
1531                          stubs.expect_container_request_uuid + '\n')
1532         self.assertEqual(exited, 0)
1533
1534
1535 class TestCreateWorkflow(unittest.TestCase):
1536     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1537     expect_workflow = StripYAMLComments(
1538         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1539     expect_workflow_altname = StripYAMLComments(
1540         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1541
1542     def setUp(self):
1543         cwltool.process._names = set()
1544         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1545
1546     def tearDown(self):
1547         root_logger = logging.getLogger('')
1548
1549         # Remove existing RuntimeStatusLoggingHandlers if they exist
1550         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1551         root_logger.handlers = handlers
1552
1553     @stubs()
1554     def test_create(self, stubs):
1555         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1556         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1557
1558         exited = arvados_cwl.main(
1559             ["--create-workflow", "--debug",
1560              "--api=containers",
1561              "--project-uuid", project_uuid,
1562              "--disable-git",
1563              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1564             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1565
1566         stubs.api.pipeline_templates().create.refute_called()
1567         stubs.api.container_requests().create.refute_called()
1568
1569         body = {
1570             "workflow": {
1571                 "owner_uuid": project_uuid,
1572                 "name": "submit_wf.cwl",
1573                 "description": "",
1574                 "definition": self.expect_workflow,
1575             }
1576         }
1577         stubs.api.workflows().create.assert_called_with(
1578             body=JsonDiffMatcher(body))
1579
1580         self.assertEqual(stubs.capture_stdout.getvalue(),
1581                          stubs.expect_workflow_uuid + '\n')
1582         self.assertEqual(exited, 0)
1583
1584     @stubs()
1585     def test_create_name(self, stubs):
1586         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1587         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1588
1589         exited = arvados_cwl.main(
1590             ["--create-workflow", "--debug",
1591              "--api=containers",
1592              "--project-uuid", project_uuid,
1593              "--name", "testing 123",
1594              "--disable-git",
1595              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1596             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1597
1598         stubs.api.pipeline_templates().create.refute_called()
1599         stubs.api.container_requests().create.refute_called()
1600
1601         body = {
1602             "workflow": {
1603                 "owner_uuid": project_uuid,
1604                 "name": "testing 123",
1605                 "description": "",
1606                 "definition": self.expect_workflow_altname,
1607             }
1608         }
1609         stubs.api.workflows().create.assert_called_with(
1610             body=JsonDiffMatcher(body))
1611
1612         self.assertEqual(stubs.capture_stdout.getvalue(),
1613                          stubs.expect_workflow_uuid + '\n')
1614         self.assertEqual(exited, 0)
1615
1616
1617     @stubs()
1618     def test_update(self, stubs):
1619         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1620         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1621
1622         exited = arvados_cwl.main(
1623             ["--update-workflow", self.existing_workflow_uuid,
1624              "--debug",
1625              "--disable-git",
1626              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1627             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1628
1629         body = {
1630             "workflow": {
1631                 "name": "submit_wf.cwl",
1632                 "description": "",
1633                 "definition": self.expect_workflow,
1634                 "owner_uuid": project_uuid
1635             }
1636         }
1637         stubs.api.workflows().update.assert_called_with(
1638             uuid=self.existing_workflow_uuid,
1639             body=JsonDiffMatcher(body))
1640         self.assertEqual(stubs.capture_stdout.getvalue(),
1641                          self.existing_workflow_uuid + '\n')
1642         self.assertEqual(exited, 0)
1643
1644
1645     @stubs()
1646     def test_update_name(self, stubs):
1647         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1648         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1649
1650         exited = arvados_cwl.main(
1651             ["--update-workflow", self.existing_workflow_uuid,
1652              "--debug", "--name", "testing 123",
1653              "--disable-git",
1654              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1655             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1656
1657         body = {
1658             "workflow": {
1659                 "name": "testing 123",
1660                 "description": "",
1661                 "definition": self.expect_workflow_altname,
1662                 "owner_uuid": project_uuid
1663             }
1664         }
1665         stubs.api.workflows().update.assert_called_with(
1666             uuid=self.existing_workflow_uuid,
1667             body=JsonDiffMatcher(body))
1668         self.assertEqual(stubs.capture_stdout.getvalue(),
1669                          self.existing_workflow_uuid + '\n')
1670         self.assertEqual(exited, 0)
1671
1672     @stubs()
1673     def test_create_collection_per_tool(self, stubs):
1674         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1675         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1676
1677         exited = arvados_cwl.main(
1678             ["--create-workflow", "--debug",
1679              "--api=containers",
1680              "--project-uuid", project_uuid,
1681              "--disable-git",
1682              "tests/collection_per_tool/collection_per_tool.cwl"],
1683             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1684
1685         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1686         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1687
1688         body = {
1689             "workflow": {
1690                 "owner_uuid": project_uuid,
1691                 "name": "collection_per_tool.cwl",
1692                 "description": "",
1693                 "definition": expect_workflow,
1694             }
1695         }
1696         stubs.api.workflows().create.assert_called_with(
1697             body=JsonDiffMatcher(body))
1698
1699         self.assertEqual(stubs.capture_stdout.getvalue(),
1700                          stubs.expect_workflow_uuid + '\n')
1701         self.assertEqual(exited, 0)
1702
1703     @stubs()
1704     def test_create_with_imports(self, stubs):
1705         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1706         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1707
1708         exited = arvados_cwl.main(
1709             ["--create-workflow", "--debug",
1710              "--api=containers",
1711              "--project-uuid", project_uuid,
1712              "tests/wf/feddemo/feddemo.cwl"],
1713             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1714
1715         stubs.api.pipeline_templates().create.refute_called()
1716         stubs.api.container_requests().create.refute_called()
1717
1718         self.assertEqual(stubs.capture_stdout.getvalue(),
1719                          stubs.expect_workflow_uuid + '\n')
1720         self.assertEqual(exited, 0)
1721
1722     @stubs()
1723     def test_create_with_no_input(self, stubs):
1724         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1725         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1726
1727         exited = arvados_cwl.main(
1728             ["--create-workflow", "--debug",
1729              "--api=containers",
1730              "--project-uuid", project_uuid,
1731              "tests/wf/revsort/revsort.cwl"],
1732             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1733
1734         stubs.api.pipeline_templates().create.refute_called()
1735         stubs.api.container_requests().create.refute_called()
1736
1737         self.assertEqual(stubs.capture_stdout.getvalue(),
1738                          stubs.expect_workflow_uuid + '\n')
1739         self.assertEqual(exited, 0)
1740
1741     @stubs()
1742     def test_create_map(self, stubs):
1743         # test uploading a document that uses objects instead of arrays
1744         # for certain fields like inputs and requirements.
1745
1746         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1747         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1748
1749         exited = arvados_cwl.main(
1750             ["--create-workflow", "--debug",
1751              "--api=containers",
1752              "--project-uuid", project_uuid,
1753              "--disable-git",
1754              "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
1755             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1756
1757         stubs.api.pipeline_templates().create.refute_called()
1758         stubs.api.container_requests().create.refute_called()
1759
1760         expect_workflow = StripYAMLComments(
1761             open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
1762
1763         body = {
1764             "workflow": {
1765                 "owner_uuid": project_uuid,
1766                 "name": "submit_wf_map.cwl",
1767                 "description": "",
1768                 "definition": expect_workflow,
1769             }
1770         }
1771         stubs.api.workflows().create.assert_called_with(
1772             body=JsonDiffMatcher(body))
1773
1774         self.assertEqual(stubs.capture_stdout.getvalue(),
1775                          stubs.expect_workflow_uuid + '\n')
1776         self.assertEqual(exited, 0)
1777
1778
1779 class TestPrintKeepDeps(unittest.TestCase):
1780     @stubs()
1781     def test_print_keep_deps(self, stubs):
1782         # test --print-keep-deps which is used by arv-copy
1783
1784         exited = arvados_cwl.main(
1785             ["--print-keep-deps", "--debug",
1786              "tests/wf/submit_wf_map.cwl"],
1787             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1788
1789         self.assertEqual(stubs.capture_stdout.getvalue(),
1790                          '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
1791         self.assertEqual(exited, 0)