sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7
   8 import copy
   9 import io
  10 import itertools
  11 import functools
  12 import hashlib
  13 import json
  14 import logging
  15 import sys
  16 import unittest
  17 import cwltool.process
  18 import re
  19 import os
  20
  21 from io import BytesIO, StringIO
  22 from unittest import mock
  23
  24 import arvados
  25 import arvados.collection
  26 import arvados_cwl
  27 import arvados_cwl.executor
  28 import arvados_cwl.runner
  29 import arvados.keep
  30
  31 from .matcher import JsonDiffMatcher, StripYAMLComments
  32 from .mock_discovery import get_rootDesc
  33
  34 import ruamel.yaml
  35
  36 _rootDesc = None
  37
  38 def stubs(wfdetails=('submit_wf.cwl', None)):
  39     def outer_wrapper(func, *rest):
  40         @functools.wraps(func)
  41         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  42         @mock.patch("uuid.uuid4")
  43         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  44         @mock.patch("arvados.collection.KeepClient")
  45         @mock.patch("arvados.keep.KeepClient")
  46         @mock.patch("arvados.events.subscribe")
  47         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  48                     uuid4, determine_image_id, *args, **kwargs):
  49             class Stubs(object):
  50                 pass
  51
  52             wfname = wfdetails[0]
  53             wfpath = wfdetails[1]
  54
  55             stubs = Stubs()
  56             stubs.events = events
  57             stubs.keepdocker = keepdocker
  58
  59             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  60                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  61                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  62                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  63                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  64                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  65                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  66                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  67                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  68
  69             determine_image_id.return_value = None
  70
  71             def putstub(p, **kwargs):
  72                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  73             keep_client1().put.side_effect = putstub
  74             keep_client1.put.side_effect = putstub
  75             keep_client2().put.side_effect = putstub
  76             keep_client2.put.side_effect = putstub
  77
  78             stubs.keep_client = keep_client2
  79             stubs.docker_images = {
  80                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  81                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  82                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  83                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  84             }
  85             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  86                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  87             stubs.keepdocker.side_effect = kd
  88
  89             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  90             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  91
  92             if sys.version_info[0] < 3:
  93                 stubs.capture_stdout = BytesIO()
  94             else:
  95                 stubs.capture_stdout = StringIO()
  96
  97             stubs.api = mock.MagicMock()
  98             stubs.api._rootDesc = get_rootDesc()
  99             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 100             stubs.api._rootDesc["revision"] = "20210628"
 101
 102             stubs.api.users().current().execute.return_value = {
 103                 "uuid": stubs.fake_user_uuid,
 104             }
 105             stubs.api.collections().list().execute.return_value = {"items": []}
 106             stubs.api.containers().current().execute.return_value = {
 107                 "uuid": stubs.fake_container_uuid,
 108             }
 109             stubs.api.config()["StorageClasses"].items.return_value = {
 110                 "default": {
 111                     "Default": True
 112                 }
 113             }.items()
 114
 115             class CollectionExecute(object):
 116                 def __init__(self, exe):
 117                     self.exe = exe
 118                 def execute(self, num_retries=None):
 119                     return self.exe
 120
 121             def collection_createstub(created_collections, body, ensure_unique_name=None):
 122                 mt = body["manifest_text"].encode('utf-8')
 123                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 124                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 125                 created_collections[uuid] = {
 126                     "uuid": uuid,
 127                     "portable_data_hash": pdh,
 128                     "manifest_text": mt.decode('utf-8')
 129                 }
 130                 return CollectionExecute(created_collections[uuid])
 131
 132             def collection_getstub(created_collections, uuid):
 133                 for v in created_collections.values():
 134                     if uuid in (v["uuid"], v["portable_data_hash"]):
 135                         return CollectionExecute(v)
 136
 137             created_collections = {
 138                 "99999999999999999999999999999998+99": {
 139                     "uuid": "",
 140                     "portable_data_hash": "99999999999999999999999999999998+99",
 141                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 142                 },
 143                 "99999999999999999999999999999997+99": {
 144                     "uuid": "",
 145                     "portable_data_hash": "99999999999999999999999999999997+99",
 146                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 147                 },
 148                 "99999999999999999999999999999994+99": {
 149                     "uuid": "",
 150                     "portable_data_hash": "99999999999999999999999999999994+99",
 151                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 152                 },
 153                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 154                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 155                     "portable_data_hash": "999999999999999999999999999999d3+99",
 156                     "manifest_text": ""
 157                 },
 158                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 159                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 160                     "portable_data_hash": "999999999999999999999999999999d4+99",
 161                     "manifest_text": ""
 162                 },
 163                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 164                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 165                     "portable_data_hash": "999999999999999999999999999999d5+99",
 166                     "manifest_text": ""
 167                 },
 168                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 169                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 170                     "portable_data_hash": "999999999999999999999999999999d6+99",
 171                     "manifest_text": ""
 172                 }
 173             }
 174             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 175             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 176
 177             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 178             stubs.api.container_requests().create().execute.return_value = {
 179                 "uuid": stubs.expect_container_request_uuid,
 180                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 181                 "state": "Queued"
 182             }
 183
 184             cwd = os.getcwd()
 185             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 186             with open(filepath) as f:
 187                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 188                 expect_packed_workflow = yaml.load(f)
 189
 190             if wfpath is None:
 191                 wfpath = wfname
 192
 193             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 194             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 195             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 196
 197             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 198             expect_packed_workflow.update(stubs.git_info)
 199
 200             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 201
 202             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 203             if wfname == wfpath:
 204                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 205             else:
 206                 container_name = wfname
 207
 208             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 209             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 210
 211             stubs.expect_container_spec = {
 212                 'priority': 500,
 213                 'mounts': {
 214                     '/var/spool/cwl': {
 215                         'writable': True,
 216                         'kind': 'collection'
 217                     },
 218                     '/var/lib/cwl/workflow.json': {
 219                         'content': expect_packed_workflow,
 220                         'kind': 'json'
 221                     },
 222                     'stdout': {
 223                         'path': '/var/spool/cwl/cwl.output.json',
 224                         'kind': 'file'
 225                     },
 226                     '/var/lib/cwl/cwl.input.json': {
 227                         'kind': 'json',
 228                         'content': {
 229                             'y': {
 230                                 'basename': '99999999999999999999999999999998+99',
 231                                 'location': 'keep:99999999999999999999999999999998+99',
 232                                 'class': 'Directory'},
 233                             'x': {
 234                                 'basename': u'blorp.txt',
 235                                 'class': 'File',
 236                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 237                                 "size": 16
 238                             },
 239                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 240                                 {'basename': 'renamed.txt',
 241                                  'class': 'File',
 242                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 243                                  'size': 0
 244                                 }
 245                             ]}
 246                         },
 247                         'kind': 'json'
 248                     }
 249                 },
 250                 'secret_mounts': {},
 251                 'state': 'Committed',
 252                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 253                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 254                             '--eval-timeout=20', '--thread-count=0',
 255                             '--enable-reuse', "--collection-cache-size=256",
 256                             '--output-name=Output from workflow '+container_name,
 257                             '--debug', '--on-error=continue',
 258                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 259                 'name': container_name,
 260                 'container_image': '999999999999999999999999999999d3+99',
 261                 'output_name': 'Output from workflow %s' % (container_name),
 262                 'output_path': '/var/spool/cwl',
 263                 'cwd': '/var/spool/cwl',
 264                 'runtime_constraints': {
 265                     'API': True,
 266                     'vcpus': 1,
 267                     'ram': (1024+256)*1024*1024
 268                 },
 269                 'properties': stubs.git_props,
 270                 'use_existing': False,
 271                 'secret_mounts': {}
 272             }
 273
 274             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 275             stubs.api.workflows().create().execute.return_value = {
 276                 "uuid": stubs.expect_workflow_uuid,
 277             }
 278             def update_mock(**kwargs):
 279                 stubs.updated_uuid = kwargs.get('uuid')
 280                 return mock.DEFAULT
 281             stubs.api.workflows().update.side_effect = update_mock
 282             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 283                 "uuid": stubs.updated_uuid,
 284             }
 285
 286             return func(self, stubs, *args, **kwargs)
 287         return wrapped
 288     return outer_wrapper
 289
 290 class TestSubmit(unittest.TestCase):
 291
 292     def setUp(self):
 293         cwltool.process._names = set()
 294         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 295
 296     def tearDown(self):
 297         root_logger = logging.getLogger('')
 298
 299         # Remove existing RuntimeStatusLoggingHandlers if they exist
 300         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 301         root_logger.handlers = handlers
 302
 303     @mock.patch("time.sleep")
 304     @stubs()
 305     def test_submit_invalid_runner_ram(self, stubs, tm):
 306         exited = arvados_cwl.main(
 307             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 308              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 309             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 310         self.assertEqual(exited, 1)
 311
 312
 313     @stubs()
 314     def test_submit_container(self, stubs):
 315         exited = arvados_cwl.main(
 316             ["--submit", "--no-wait", "--api=containers", "--debug",
 317                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 318             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 319
 320         stubs.api.collections().create.assert_has_calls([
 321             mock.call(body=JsonDiffMatcher({
 322                 'manifest_text':
 323                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 324                 'replication_desired': None,
 325                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 326             }), ensure_unique_name=False),
 327             mock.call(body=JsonDiffMatcher({
 328                 'manifest_text':
 329                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 330                 'replication_desired': None,
 331                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 332             }), ensure_unique_name=False),
 333             ])
 334
 335         expect_container = copy.deepcopy(stubs.expect_container_spec)
 336         stubs.api.container_requests().create.assert_called_with(
 337             body=JsonDiffMatcher(expect_container))
 338         self.assertEqual(stubs.capture_stdout.getvalue(),
 339                          stubs.expect_container_request_uuid + '\n')
 340         self.assertEqual(exited, 0)
 341
 342
 343     @stubs()
 344     def test_submit_container_tool(self, stubs):
 345         # test for issue #16139
 346         exited = arvados_cwl.main(
 347             ["--submit", "--no-wait", "--api=containers", "--debug",
 348                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 349             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 350
 351         self.assertEqual(stubs.capture_stdout.getvalue(),
 352                          stubs.expect_container_request_uuid + '\n')
 353         self.assertEqual(exited, 0)
 354
 355     @stubs()
 356     def test_submit_container_no_reuse(self, stubs):
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 359                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 361
 362         expect_container = copy.deepcopy(stubs.expect_container_spec)
 363         expect_container["command"] = [
 364             'arvados-cwl-runner', '--local', '--api=containers',
 365             '--no-log-timestamps', '--disable-validate', '--disable-color',
 366             '--eval-timeout=20', '--thread-count=0',
 367             '--disable-reuse', "--collection-cache-size=256",
 368             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 369             '--debug', '--on-error=continue',
 370             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 371         expect_container["use_existing"] = False
 372
 373         stubs.api.container_requests().create.assert_called_with(
 374             body=JsonDiffMatcher(expect_container))
 375         self.assertEqual(stubs.capture_stdout.getvalue(),
 376                          stubs.expect_container_request_uuid + '\n')
 377         self.assertEqual(exited, 0)
 378
 379     @stubs(('submit_wf_no_reuse.cwl', None))
 380     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 381         exited = arvados_cwl.main(
 382             ["--submit", "--no-wait", "--api=containers", "--debug",
 383              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 384             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 385         self.assertEqual(exited, 0)
 386
 387         expect_container = copy.deepcopy(stubs.expect_container_spec)
 388         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 389         expect_container["use_existing"] = False
 390         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 391             {
 392                 "class": "WorkReuse",
 393                 "enableReuse": False,
 394             },
 395                 {
 396                     "acrContainerImage": "999999999999999999999999999999d3+99",
 397                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 398                 }
 399         ]
 400         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 401
 402         stubs.api.container_requests().create.assert_called_with(
 403             body=JsonDiffMatcher(expect_container))
 404         self.assertEqual(stubs.capture_stdout.getvalue(),
 405                          stubs.expect_container_request_uuid + '\n')
 406
 407
 408     @stubs()
 409     def test_submit_container_on_error(self, stubs):
 410         exited = arvados_cwl.main(
 411             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 412                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 413             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 414
 415         expect_container = copy.deepcopy(stubs.expect_container_spec)
 416         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 417                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 418                                        '--eval-timeout=20', '--thread-count=0',
 419                                        '--enable-reuse', "--collection-cache-size=256",
 420                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 421                                        '--debug', '--on-error=stop',
 422                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 423
 424         stubs.api.container_requests().create.assert_called_with(
 425             body=JsonDiffMatcher(expect_container))
 426         self.assertEqual(stubs.capture_stdout.getvalue(),
 427                          stubs.expect_container_request_uuid + '\n')
 428         self.assertEqual(exited, 0)
 429
 430     @stubs()
 431     def test_submit_container_output_name(self, stubs):
 432         output_name = "test_output_name"
 433
 434         exited = arvados_cwl.main(
 435             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 436                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 437             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 438
 439         expect_container = copy.deepcopy(stubs.expect_container_spec)
 440         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 441                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 442                                        '--eval-timeout=20', '--thread-count=0',
 443                                        '--enable-reuse', "--collection-cache-size=256",
 444                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 445                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 446         expect_container["output_name"] = output_name
 447
 448         stubs.api.container_requests().create.assert_called_with(
 449             body=JsonDiffMatcher(expect_container))
 450         self.assertEqual(stubs.capture_stdout.getvalue(),
 451                          stubs.expect_container_request_uuid + '\n')
 452         self.assertEqual(exited, 0)
 453
 454     @stubs()
 455     def test_submit_storage_classes(self, stubs):
 456         exited = arvados_cwl.main(
 457             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 458                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 459             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 460
 461         expect_container = copy.deepcopy(stubs.expect_container_spec)
 462         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 463                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 464                                        '--eval-timeout=20', '--thread-count=0',
 465                                        '--enable-reuse', "--collection-cache-size=256",
 466                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 467                                        "--debug",
 468                                        "--storage-classes=foo", '--on-error=continue',
 469                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 470
 471         stubs.api.container_requests().create.assert_called_with(
 472             body=JsonDiffMatcher(expect_container))
 473         self.assertEqual(stubs.capture_stdout.getvalue(),
 474                          stubs.expect_container_request_uuid + '\n')
 475         self.assertEqual(exited, 0)
 476
 477     @stubs()
 478     def test_submit_multiple_storage_classes(self, stubs):
 479         exited = arvados_cwl.main(
 480             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 481                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 482             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 483
 484         expect_container = copy.deepcopy(stubs.expect_container_spec)
 485         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 486                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 487                                        '--eval-timeout=20', '--thread-count=0',
 488                                        '--enable-reuse', "--collection-cache-size=256",
 489                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 490                                        "--debug",
 491                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 492                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 493
 494         stubs.api.container_requests().create.assert_called_with(
 495             body=JsonDiffMatcher(expect_container))
 496         self.assertEqual(stubs.capture_stdout.getvalue(),
 497                          stubs.expect_container_request_uuid + '\n')
 498         self.assertEqual(exited, 0)
 499
 500     @mock.patch("cwltool.task_queue.TaskQueue")
 501     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 502     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 503     @stubs()
 504     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 505         final_output_c = arvados.collection.Collection()
 506         make_output.return_value = ({},final_output_c)
 507
 508         def set_final_output(job_order, output_callback, runtimeContext):
 509             output_callback({"out": "zzzzz"}, "success")
 510             return []
 511         job.side_effect = set_final_output
 512
 513         exited = arvados_cwl.main(
 514             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 515                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 516             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 517
 518         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 519         self.assertEqual(exited, 0)
 520
 521     @mock.patch("cwltool.task_queue.TaskQueue")
 522     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 523     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 524     @stubs()
 525     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 526         final_output_c = arvados.collection.Collection()
 527         make_output.return_value = ({},final_output_c)
 528         stubs.api.config().get.return_value = {"default": {"Default": True}}
 529
 530         def set_final_output(job_order, output_callback, runtimeContext):
 531             output_callback({"out": "zzzzz"}, "success")
 532             return []
 533         job.side_effect = set_final_output
 534
 535         exited = arvados_cwl.main(
 536             ["--debug", "--local", "--disable-git",
 537                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 538             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 539
 540         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 541         self.assertEqual(exited, 0)
 542
 543     @mock.patch("cwltool.task_queue.TaskQueue")
 544     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 545     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 546     @stubs()
 547     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 548         final_output_c = arvados.collection.Collection()
 549         make_output.return_value = ({},final_output_c)
 550
 551         def set_final_output(job_order, output_callback, runtimeContext):
 552             output_callback({"out": "zzzzz"}, "success")
 553             return []
 554         job.side_effect = set_final_output
 555
 556         exited = arvados_cwl.main(
 557             ["--debug", "--local", "--disable-git",
 558                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 559             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 560
 561         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 562         self.assertEqual(exited, 0)
 563
 564     @stubs()
 565     def test_submit_container_output_ttl(self, stubs):
 566         exited = arvados_cwl.main(
 567             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 568                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 569             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 570
 571         expect_container = copy.deepcopy(stubs.expect_container_spec)
 572         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 573                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 574                                        '--eval-timeout=20', '--thread-count=0',
 575                                        '--enable-reuse', "--collection-cache-size=256",
 576                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 577                                        '--debug',
 578                                        '--on-error=continue',
 579                                        "--intermediate-output-ttl=3600",
 580                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 581
 582         stubs.api.container_requests().create.assert_called_with(
 583             body=JsonDiffMatcher(expect_container))
 584         self.assertEqual(stubs.capture_stdout.getvalue(),
 585                          stubs.expect_container_request_uuid + '\n')
 586         self.assertEqual(exited, 0)
 587
 588     @stubs()
 589     def test_submit_container_trash_intermediate(self, stubs):
 590         exited = arvados_cwl.main(
 591             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 592                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 593             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 594
 595
 596         expect_container = copy.deepcopy(stubs.expect_container_spec)
 597         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 598                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 599                                        '--eval-timeout=20', '--thread-count=0',
 600                                        '--enable-reuse', "--collection-cache-size=256",
 601                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 602                                        '--debug', '--on-error=continue',
 603                                        "--trash-intermediate",
 604                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 605
 606         stubs.api.container_requests().create.assert_called_with(
 607             body=JsonDiffMatcher(expect_container))
 608         self.assertEqual(stubs.capture_stdout.getvalue(),
 609                          stubs.expect_container_request_uuid + '\n')
 610         self.assertEqual(exited, 0)
 611
 612     @stubs()
 613     def test_submit_container_output_tags(self, stubs):
 614         output_tags = "tag0,tag1,tag2"
 615
 616         exited = arvados_cwl.main(
 617             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 618                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 619             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 620
 621         expect_container = copy.deepcopy(stubs.expect_container_spec)
 622         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 623                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 624                                        '--eval-timeout=20', '--thread-count=0',
 625                                        '--enable-reuse', "--collection-cache-size=256",
 626                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 627                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 628                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 629
 630         stubs.api.container_requests().create.assert_called_with(
 631             body=JsonDiffMatcher(expect_container))
 632         self.assertEqual(stubs.capture_stdout.getvalue(),
 633                          stubs.expect_container_request_uuid + '\n')
 634         self.assertEqual(exited, 0)
 635
 636     @stubs()
 637     def test_submit_container_runner_ram(self, stubs):
 638         exited = arvados_cwl.main(
 639             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 640                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 641             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 642
 643         expect_container = copy.deepcopy(stubs.expect_container_spec)
 644         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 645         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 646                 {
 647                     "acrContainerImage": "999999999999999999999999999999d3+99",
 648                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 649                     "ramMin": 2048
 650                 }
 651         ]
 652
 653         stubs.api.container_requests().create.assert_called_with(
 654             body=JsonDiffMatcher(expect_container))
 655         self.assertEqual(stubs.capture_stdout.getvalue(),
 656                          stubs.expect_container_request_uuid + '\n')
 657         self.assertEqual(exited, 0)
 658
 659     @mock.patch("arvados.collection.CollectionReader")
 660     @mock.patch("time.sleep")
 661     @stubs()
 662     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 663         collectionReader().exists.return_value = True
 664         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 665         exited = arvados_cwl.main(
 666             ["--submit", "--no-wait", "--api=containers", "--debug",
 667              "tests/wf/submit_keepref_wf.cwl"],
 668             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 669         self.assertEqual(exited, 0)
 670
 671     @mock.patch("arvados.collection.CollectionReader")
 672     @mock.patch("time.sleep")
 673     @stubs()
 674     def test_submit_keepref(self, stubs, tm, reader):
 675         with open("tests/wf/expect_arvworkflow.cwl") as f:
 676             reader().open().__enter__().read.return_value = f.read()
 677
 678         exited = arvados_cwl.main(
 679             ["--submit", "--no-wait", "--api=containers", "--debug",
 680              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 681             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 682
 683         expect_container = {
 684             'priority': 500,
 685             'mounts': {
 686                 '/var/spool/cwl': {
 687                     'writable': True,
 688                     'kind': 'collection'
 689                 },
 690                 'stdout': {
 691                     'path': '/var/spool/cwl/cwl.output.json',
 692                     'kind': 'file'
 693                 },
 694                 '/var/lib/cwl/workflow': {
 695                     'portable_data_hash': '99999999999999999999999999999994+99',
 696                     'kind': 'collection'
 697                 },
 698                 '/var/lib/cwl/cwl.input.json': {
 699                     'content': {
 700                         'x': 'XxX'
 701                     },
 702                     'kind': 'json'
 703                 }
 704             }, 'state': 'Committed',
 705             'output_path': '/var/spool/cwl',
 706             'name': 'expect_arvworkflow.cwl#main',
 707             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 708             'container_image': '999999999999999999999999999999d3+99',
 709             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 710                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 711                         '--eval-timeout=20', '--thread-count=0',
 712                         '--enable-reuse', "--collection-cache-size=256",
 713                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 714                         '--debug', '--on-error=continue',
 715                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 716             'cwd': '/var/spool/cwl',
 717             'runtime_constraints': {
 718                 'API': True,
 719                 'vcpus': 1,
 720                 'ram': 1342177280
 721             },
 722             'use_existing': False,
 723             'properties': {},
 724             'secret_mounts': {}
 725         }
 726
 727         stubs.api.container_requests().create.assert_called_with(
 728             body=JsonDiffMatcher(expect_container))
 729         self.assertEqual(stubs.capture_stdout.getvalue(),
 730                          stubs.expect_container_request_uuid + '\n')
 731         self.assertEqual(exited, 0)
 732
 733     @mock.patch("time.sleep")
 734     @stubs()
 735     def test_submit_arvworkflow(self, stubs, tm):
 736         with open("tests/wf/expect_arvworkflow.cwl") as f:
 737             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 738
 739         exited = arvados_cwl.main(
 740             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 741              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 742             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 743
 744         expect_container = {
 745             'priority': 500,
 746             'mounts': {
 747                 '/var/spool/cwl': {
 748                     'writable': True,
 749                     'kind': 'collection'
 750                 },
 751                 'stdout': {
 752                     'path': '/var/spool/cwl/cwl.output.json',
 753                     'kind': 'file'
 754                 },
 755                 '/var/lib/cwl/workflow.json': {
 756                     'kind': 'json',
 757                     'content': {
 758                         'cwlVersion': 'v1.0',
 759                         'label': 'a test workflow',
 760                         '$graph': [
 761                             {
 762                                 'id': '#main',
 763                                 'inputs': [
 764                                     {'type': 'string', 'id': '#main/x'}
 765                                 ],
 766                                 'steps': [
 767                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 768                                      'run': '#submit_tool.cwl',
 769                                      'id': '#main/step1',
 770                                      'out': []}
 771                                 ],
 772                                 'class': 'Workflow',
 773                                 'outputs': []
 774                             },
 775                             {
 776                                 'inputs': [
 777                                     {
 778                                         'inputBinding': {'position': 1},
 779                                         'type': 'string',
 780                                         'id': '#submit_tool.cwl/x'}
 781                                 ],
 782                                 'requirements': [
 783                                     {
 784                                         'dockerPull': 'debian:buster-slim',
 785                                         'class': 'DockerRequirement'
 786                                     }
 787                                 ],
 788                                 'id': '#submit_tool.cwl',
 789                                 'outputs': [],
 790                                 'baseCommand': 'cat',
 791                                 'class': 'CommandLineTool'
 792                             }
 793                         ]
 794                     }
 795                 },
 796                 '/var/lib/cwl/cwl.input.json': {
 797                     'content': {
 798                         'x': 'XxX'
 799                     },
 800                     'kind': 'json'
 801                 }
 802             }, 'state': 'Committed',
 803             'output_path': '/var/spool/cwl',
 804             'name': 'a test workflow',
 805             'container_image': "999999999999999999999999999999d3+99",
 806             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 807                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 808                         '--eval-timeout=20', '--thread-count=0',
 809                         '--enable-reuse', "--collection-cache-size=256",
 810                         "--output-name=Output from workflow a test workflow",
 811                         '--debug', '--on-error=continue',
 812                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 813             'output_name': 'Output from workflow a test workflow',
 814             'cwd': '/var/spool/cwl',
 815             'runtime_constraints': {
 816                 'API': True,
 817                 'vcpus': 1,
 818                 'ram': 1342177280
 819             },
 820             'use_existing': False,
 821             'properties': {
 822                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 823             },
 824             'secret_mounts': {}
 825         }
 826
 827         stubs.api.container_requests().create.assert_called_with(
 828             body=JsonDiffMatcher(expect_container))
 829         self.assertEqual(stubs.capture_stdout.getvalue(),
 830                          stubs.expect_container_request_uuid + '\n')
 831         self.assertEqual(exited, 0)
 832
 833     @stubs(('hello container 123', 'submit_wf.cwl'))
 834     def test_submit_container_name(self, stubs):
 835         exited = arvados_cwl.main(
 836             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 837                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 838             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 839
 840         expect_container = copy.deepcopy(stubs.expect_container_spec)
 841
 842         stubs.api.container_requests().create.assert_called_with(
 843             body=JsonDiffMatcher(expect_container))
 844         self.assertEqual(stubs.capture_stdout.getvalue(),
 845                          stubs.expect_container_request_uuid + '\n')
 846         self.assertEqual(exited, 0)
 847
 848     @stubs()
 849     def test_submit_missing_input(self, stubs):
 850         exited = arvados_cwl.main(
 851             ["--submit", "--no-wait", "--api=containers", "--debug",
 852              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 853             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 854         self.assertEqual(exited, 0)
 855
 856         exited = arvados_cwl.main(
 857             ["--submit", "--no-wait", "--api=containers", "--debug",
 858              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 859             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 860         self.assertEqual(exited, 1)
 861
 862     @stubs()
 863     def test_submit_container_project(self, stubs):
 864         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 865         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 866         exited = arvados_cwl.main(
 867             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 868                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 869             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 870
 871         expect_container = copy.deepcopy(stubs.expect_container_spec)
 872         expect_container["owner_uuid"] = project_uuid
 873         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 874                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 875                                        "--eval-timeout=20", "--thread-count=0",
 876                                        '--enable-reuse', "--collection-cache-size=256",
 877                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 878                                        '--debug',
 879                                        '--on-error=continue',
 880                                        '--project-uuid='+project_uuid,
 881                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 882
 883         stubs.api.container_requests().create.assert_called_with(
 884             body=JsonDiffMatcher(expect_container))
 885         self.assertEqual(stubs.capture_stdout.getvalue(),
 886                          stubs.expect_container_request_uuid + '\n')
 887         self.assertEqual(exited, 0)
 888
 889     @stubs()
 890     def test_submit_container_eval_timeout(self, stubs):
 891         exited = arvados_cwl.main(
 892             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 893                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 894             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 895
 896         expect_container = copy.deepcopy(stubs.expect_container_spec)
 897         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 898                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 899                                        '--eval-timeout=60.0', '--thread-count=0',
 900                                        '--enable-reuse', "--collection-cache-size=256",
 901                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 902                                        '--debug', '--on-error=continue',
 903                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 904
 905         stubs.api.container_requests().create.assert_called_with(
 906             body=JsonDiffMatcher(expect_container))
 907         self.assertEqual(stubs.capture_stdout.getvalue(),
 908                          stubs.expect_container_request_uuid + '\n')
 909         self.assertEqual(exited, 0)
 910
 911     @stubs()
 912     def test_submit_container_collection_cache(self, stubs):
 913         exited = arvados_cwl.main(
 914             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 915                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 916             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 917
 918         expect_container = copy.deepcopy(stubs.expect_container_spec)
 919         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 920                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 921                                        '--eval-timeout=20', '--thread-count=0',
 922                                        '--enable-reuse', "--collection-cache-size=500",
 923                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 924                                        '--debug', '--on-error=continue',
 925                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 926         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 927
 928         stubs.api.container_requests().create.assert_called_with(
 929             body=JsonDiffMatcher(expect_container))
 930         self.assertEqual(stubs.capture_stdout.getvalue(),
 931                          stubs.expect_container_request_uuid + '\n')
 932         self.assertEqual(exited, 0)
 933
 934     @stubs()
 935     def test_submit_container_thread_count(self, stubs):
 936         exited = arvados_cwl.main(
 937             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 938                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 939             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 940
 941         expect_container = copy.deepcopy(stubs.expect_container_spec)
 942         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 943                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 944                                        '--eval-timeout=20', '--thread-count=20',
 945                                        '--enable-reuse', "--collection-cache-size=256",
 946                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 947                                        '--debug', '--on-error=continue',
 948                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 949
 950         stubs.api.container_requests().create.assert_called_with(
 951             body=JsonDiffMatcher(expect_container))
 952         self.assertEqual(stubs.capture_stdout.getvalue(),
 953                          stubs.expect_container_request_uuid + '\n')
 954         self.assertEqual(exited, 0)
 955
 956     @stubs()
 957     def test_submit_container_runner_image(self, stubs):
 958         exited = arvados_cwl.main(
 959             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 960                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 961             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 962
 963         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 964         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 965                 {
 966                     "acrContainerImage": "999999999999999999999999999999d5+99",
 967                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 968                 }
 969         ]
 970
 971         expect_container = copy.deepcopy(stubs.expect_container_spec)
 972         stubs.api.container_requests().create.assert_called_with(
 973             body=JsonDiffMatcher(expect_container))
 974         self.assertEqual(stubs.capture_stdout.getvalue(),
 975                          stubs.expect_container_request_uuid + '\n')
 976         self.assertEqual(exited, 0)
 977
 978     @stubs()
 979     def test_submit_priority(self, stubs):
 980         exited = arvados_cwl.main(
 981             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 982                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 983             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 984
 985         stubs.expect_container_spec["priority"] = 669
 986
 987         expect_container = copy.deepcopy(stubs.expect_container_spec)
 988         stubs.api.container_requests().create.assert_called_with(
 989             body=JsonDiffMatcher(expect_container))
 990         self.assertEqual(stubs.capture_stdout.getvalue(),
 991                          stubs.expect_container_request_uuid + '\n')
 992         self.assertEqual(exited, 0)
 993
 994     @stubs(('submit_wf_runner_resources.cwl', None))
 995     def test_submit_wf_runner_resources(self, stubs):
 996         exited = arvados_cwl.main(
 997             ["--submit", "--no-wait", "--api=containers", "--debug",
 998                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 999             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1000
1001         expect_container = copy.deepcopy(stubs.expect_container_spec)
1002         expect_container["runtime_constraints"] = {
1003             "API": True,
1004             "vcpus": 2,
1005             "ram": (2000+512) * 2**20
1006         }
1007         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1008             {
1009                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1010                 "acrContainerImage": "999999999999999999999999999999d3+99",
1011                 "coresMin": 2,
1012                 "ramMin": 2000,
1013                 "keep_cache": 512
1014             }
1015         ]
1016         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1017         #    "arv": "http://arvados.org/cwl#",
1018         #}
1019         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1020         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1021
1022         stubs.api.container_requests().create.assert_called_with(
1023             body=JsonDiffMatcher(expect_container))
1024         self.assertEqual(stubs.capture_stdout.getvalue(),
1025                          stubs.expect_container_request_uuid + '\n')
1026         self.assertEqual(exited, 0)
1027
1028     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1029     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1030     @mock.patch("arvados.api")
1031     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1032         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1033
1034         arvrunner = mock.MagicMock()
1035         arvrunner.project_uuid = ""
1036         api.return_value = mock.MagicMock()
1037         arvrunner.api = api.return_value
1038         arvrunner.runtimeContext.match_local_docker = False
1039         arvrunner.api.links().list().execute.side_effect = itertools.cycle([
1040             {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
1041                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1042                         "link_class": "docker_image_repo+tag",
1043                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1044                         "owner_uuid": "",
1045                         "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
1046                         "properties": {"image_timestamp": ""}}]},
1047             {"items": []},
1048             {"items": []},
1049             {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
1050                         "head_uuid": "",
1051                         "link_class": "docker_image_hash",
1052                         "name": "123456",
1053                         "owner_uuid": "",
1054                         "uuid": "zzzzz-o0j2j-arvadosjobshash",
1055                         "properties": {"image_timestamp": ""}}]},
1056             {"items": []},
1057             {"items": []},
1058         ])
1059         find_one_image_hash.return_value = "123456"
1060
1061         arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
1062             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1063                         "owner_uuid": "",
1064                         "manifest_text": "",
1065                         "created_at": "2023-08-25T12:34:55.012345Z",
1066                         "properties": {}}]},
1067             {"items": []},
1068             {"items": []},
1069         ])
1070         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1071         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1072                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1073
1074         self.assertEqual("9999999999999999999999999999999b+99",
1075                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1076
1077
1078     @stubs()
1079     def test_submit_secrets(self, stubs):
1080         exited = arvados_cwl.main(
1081             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1082                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1083             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1084
1085         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1086         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1087
1088         expect_container = {
1089             "command": [
1090                 "arvados-cwl-runner",
1091                 "--local",
1092                 "--api=containers",
1093                 "--no-log-timestamps",
1094                 "--disable-validate",
1095                 "--disable-color",
1096                 "--eval-timeout=20",
1097                 '--thread-count=0',
1098                 "--enable-reuse",
1099                 "--collection-cache-size=256",
1100                 "--output-name=Output from workflow secret_wf.cwl",
1101                 "--debug",
1102                 "--on-error=continue",
1103                 "/var/lib/cwl/workflow.json#main",
1104                 "/var/lib/cwl/cwl.input.json"
1105             ],
1106             "container_image": "999999999999999999999999999999d3+99",
1107             "cwd": "/var/spool/cwl",
1108             "mounts": {
1109                 "/var/lib/cwl/cwl.input.json": {
1110                     "content": {
1111                         "pw": {
1112                             "$include": "/secrets/s0"
1113                         }
1114                     },
1115                     "kind": "json"
1116                 },
1117                 "/var/lib/cwl/workflow.json": {
1118                     "content": {
1119                         "$graph": [
1120                             {
1121                                 "class": "Workflow",
1122                                 "hints": [
1123                                     {
1124                                     "class": "DockerRequirement",
1125                                     "dockerPull": "debian:buster-slim",
1126                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1127                                     },
1128                                     {
1129                                         "class": "http://commonwl.org/cwltool#Secrets",
1130                                         "secrets": [
1131                                             "#main/pw"
1132                                         ]
1133                                     },
1134                                     {
1135                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1136                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1137                                     }
1138                                 ],
1139                                 "id": "#main",
1140                                 "inputs": [
1141                                     {
1142                                         "id": "#main/pw",
1143                                         "type": "string"
1144                                     }
1145                                 ],
1146                                 "outputs": [
1147                                     {
1148                                         "id": "#main/out",
1149                                         "outputSource": "#main/step/out",
1150                                         "type": "File"
1151                                     }
1152                                 ],
1153                                 "requirements": [
1154                                     {
1155                                         "class": "SubworkflowFeatureRequirement"
1156                                     }
1157                                 ],
1158                                 "steps": [
1159                                     {
1160                                         "id": "#main/secret_wf.cwl",
1161                                         "in": [
1162                                             {
1163                                                 "id": "#main/step/pw",
1164                                                 "source": "#main/pw"
1165                                             }
1166                                         ],
1167                                         "label": "secret_wf.cwl",
1168                                         "out": [
1169                                             {"id": "#main/step/out"}
1170                                         ],
1171                                         "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
1172                                     }
1173                                 ]
1174                             }
1175                         ],
1176                         "cwlVersion": "v1.2"
1177                     },
1178                     "kind": "json"
1179                 },
1180                 "/var/spool/cwl": {
1181                     "kind": "collection",
1182                     "writable": True
1183                 },
1184                 "stdout": {
1185                     "kind": "file",
1186                     "path": "/var/spool/cwl/cwl.output.json"
1187                 }
1188             },
1189             "name": "secret_wf.cwl",
1190             "output_name": "Output from workflow secret_wf.cwl",
1191             "output_path": "/var/spool/cwl",
1192             "priority": 500,
1193             "properties": {},
1194             "runtime_constraints": {
1195                 "API": True,
1196                 "ram": 1342177280,
1197                 "vcpus": 1
1198             },
1199             "secret_mounts": {
1200                 "/secrets/s0": {
1201                     "content": "blorp",
1202                     "kind": "text"
1203                 }
1204             },
1205             "state": "Committed",
1206             "use_existing": False
1207         }
1208
1209         stubs.api.container_requests().create.assert_called_with(
1210             body=JsonDiffMatcher(expect_container))
1211         self.assertEqual(stubs.capture_stdout.getvalue(),
1212                          stubs.expect_container_request_uuid + '\n')
1213         self.assertEqual(exited, 0)
1214
1215     @stubs()
1216     def test_submit_request_uuid(self, stubs):
1217         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1218         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1219
1220         stubs.api.container_requests().update().execute.return_value = {
1221             "uuid": stubs.expect_container_request_uuid,
1222             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1223             "state": "Queued"
1224         }
1225
1226         exited = arvados_cwl.main(
1227             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1228                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1229             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1230
1231         stubs.api.container_requests().update.assert_called_with(
1232             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1233         self.assertEqual(stubs.capture_stdout.getvalue(),
1234                          stubs.expect_container_request_uuid + '\n')
1235         self.assertEqual(exited, 0)
1236
1237     @stubs()
1238     def test_submit_container_cluster_id(self, stubs):
1239         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1240
1241         exited = arvados_cwl.main(
1242             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1243                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1244             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1245
1246         expect_container = copy.deepcopy(stubs.expect_container_spec)
1247
1248         stubs.api.container_requests().create.assert_called_with(
1249             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1250         self.assertEqual(stubs.capture_stdout.getvalue(),
1251                          stubs.expect_container_request_uuid + '\n')
1252         self.assertEqual(exited, 0)
1253
1254     @stubs()
1255     def test_submit_validate_cluster_id(self, stubs):
1256         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1257         exited = arvados_cwl.main(
1258             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1259              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1260             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1261         self.assertEqual(exited, 1)
1262
1263     @stubs()
1264     def test_submit_validate_project_uuid(self, stubs):
1265         # Fails with bad cluster prefix
1266         exited = arvados_cwl.main(
1267             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1268              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1269             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1270         self.assertEqual(exited, 1)
1271
1272         # Project lookup fails
1273         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1274         exited = arvados_cwl.main(
1275             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1277             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1278         self.assertEqual(exited, 1)
1279
1280         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1281         exited = arvados_cwl.main(
1282             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1283              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1284             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1285         self.assertEqual(exited, 0)
1286
1287
1288     @mock.patch("arvados.collection.CollectionReader")
1289     @stubs()
1290     def test_submit_uuid_inputs(self, stubs, collectionReader):
1291         collectionReader().exists.return_value = True
1292         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1293         def list_side_effect(**kwargs):
1294             m = mock.MagicMock()
1295             if "count" in kwargs:
1296                 m.execute.return_value = {"items": [
1297                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1298                 ]}
1299             else:
1300                 m.execute.return_value = {"items": []}
1301             return m
1302         stubs.api.collections().list.side_effect = list_side_effect
1303         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1304
1305         exited = arvados_cwl.main(
1306             ["--submit", "--no-wait", "--api=containers", "--debug",
1307                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1308             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1309
1310         expect_container = copy.deepcopy(stubs.expect_container_spec)
1311         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1312         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1313         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1314
1315         stubs.api.collections().list.assert_has_calls([
1316             mock.call(count='none',
1317                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1318                       select=['uuid', 'portable_data_hash'])])
1319         stubs.api.container_requests().create.assert_called_with(
1320             body=JsonDiffMatcher(expect_container))
1321         self.assertEqual(stubs.capture_stdout.getvalue(),
1322                          stubs.expect_container_request_uuid + '\n')
1323         self.assertEqual(exited, 0)
1324
1325     @stubs()
1326     def test_submit_mismatched_uuid_inputs(self, stubs):
1327         def list_side_effect(**kwargs):
1328             m = mock.MagicMock()
1329             if "count" in kwargs:
1330                 m.execute.return_value = {"items": [
1331                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1332                 ]}
1333             else:
1334                 m.execute.return_value = {"items": []}
1335             return m
1336         stubs.api.collections().list.side_effect = list_side_effect
1337
1338         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1339             capture_stderr = StringIO()
1340             cwltool_logger = logging.getLogger('cwltool')
1341             stderr_logger = logging.StreamHandler(capture_stderr)
1342             cwltool_logger.addHandler(stderr_logger)
1343
1344             try:
1345                 exited = arvados_cwl.main(
1346                     ["--submit", "--no-wait", "--api=containers", "--debug",
1347                         "tests/wf/submit_wf.cwl", infile],
1348                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1349
1350                 self.assertEqual(exited, 1)
1351                 self.assertRegex(
1352                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1353                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1354             finally:
1355                 cwltool_logger.removeHandler(stderr_logger)
1356
1357     @mock.patch("arvados.collection.CollectionReader")
1358     @stubs()
1359     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1360         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1361         capture_stderr = StringIO()
1362
1363         cwltool_logger = logging.getLogger('cwltool')
1364         stderr_logger = logging.StreamHandler(capture_stderr)
1365         cwltool_logger.addHandler(stderr_logger)
1366
1367         exited = arvados_cwl.main(
1368             ["--submit", "--no-wait", "--api=containers", "--debug",
1369                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1370             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1371
1372         try:
1373             self.assertEqual(exited, 1)
1374             self.assertRegex(
1375                 capture_stderr.getvalue(),
1376                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1377         finally:
1378             cwltool_logger.removeHandler(stderr_logger)
1379
1380     @stubs(('submit_wf_process_properties.cwl', None))
1381     def test_submit_set_process_properties(self, stubs):
1382         exited = arvados_cwl.main(
1383             ["--submit", "--no-wait", "--api=containers", "--debug",
1384                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1385             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1386
1387         expect_container = copy.deepcopy(stubs.expect_container_spec)
1388
1389         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1390             {
1391                 "class": "http://arvados.org/cwl#ProcessProperties",
1392                 "processProperties": [
1393                     {"propertyName": "baz",
1394                      "propertyValue": "$(inputs.x.basename)"},
1395                     {"propertyName": "foo",
1396                      "propertyValue": "bar"},
1397                     {"propertyName": "quux",
1398                      "propertyValue": {
1399                          "q1": 1,
1400                          "q2": 2
1401                      }
1402                     }
1403                 ],
1404             },
1405             {
1406                 "acrContainerImage": "999999999999999999999999999999d3+99",
1407                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1408             }
1409         ]
1410         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1411         #    "arv": "http://arvados.org/cwl#"
1412         #}
1413
1414         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1415
1416         expect_container["properties"].update({
1417             "baz": "blorp.txt",
1418             "foo": "bar",
1419             "quux": {
1420                 "q1": 1,
1421                 "q2": 2
1422             }
1423         })
1424
1425         stubs.api.container_requests().create.assert_called_with(
1426             body=JsonDiffMatcher(expect_container))
1427         self.assertEqual(stubs.capture_stdout.getvalue(),
1428                          stubs.expect_container_request_uuid + '\n')
1429         self.assertEqual(exited, 0)
1430
1431
1432     @stubs()
1433     def test_submit_enable_preemptible(self, stubs):
1434         exited = arvados_cwl.main(
1435             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1436                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1437             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1438
1439         expect_container = copy.deepcopy(stubs.expect_container_spec)
1440         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1441                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1442                                        '--eval-timeout=20', '--thread-count=0',
1443                                        '--enable-reuse', "--collection-cache-size=256",
1444                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1445                                        '--debug', '--on-error=continue',
1446                                        '--enable-preemptible',
1447                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1448
1449         stubs.api.container_requests().create.assert_called_with(
1450             body=JsonDiffMatcher(expect_container))
1451         self.assertEqual(stubs.capture_stdout.getvalue(),
1452                          stubs.expect_container_request_uuid + '\n')
1453         self.assertEqual(exited, 0)
1454
1455     @stubs()
1456     def test_submit_disable_preemptible(self, stubs):
1457         exited = arvados_cwl.main(
1458             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1459                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1460             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1461
1462         expect_container = copy.deepcopy(stubs.expect_container_spec)
1463         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1464                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1465                                        '--eval-timeout=20', '--thread-count=0',
1466                                        '--enable-reuse', "--collection-cache-size=256",
1467                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1468                                        '--debug', '--on-error=continue',
1469                                        '--disable-preemptible',
1470                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1471
1472         stubs.api.container_requests().create.assert_called_with(
1473             body=JsonDiffMatcher(expect_container))
1474         self.assertEqual(stubs.capture_stdout.getvalue(),
1475                          stubs.expect_container_request_uuid + '\n')
1476         self.assertEqual(exited, 0)
1477
1478     @stubs()
1479     def test_submit_container_prefer_cached_downloads(self, stubs):
1480         exited = arvados_cwl.main(
1481             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1482                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1483             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1484
1485         expect_container = copy.deepcopy(stubs.expect_container_spec)
1486         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1487                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1488                                        '--eval-timeout=20', '--thread-count=0',
1489                                        '--enable-reuse', "--collection-cache-size=256",
1490                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1491                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1492                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1493
1494         stubs.api.container_requests().create.assert_called_with(
1495             body=JsonDiffMatcher(expect_container))
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_container_request_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500     @stubs()
1501     def test_submit_container_varying_url_params(self, stubs):
1502         exited = arvados_cwl.main(
1503             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1504                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1505             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1506
1507         expect_container = copy.deepcopy(stubs.expect_container_spec)
1508         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1509                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1510                                        '--eval-timeout=20', '--thread-count=0',
1511                                        '--enable-reuse', "--collection-cache-size=256",
1512                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1513                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1514                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1515
1516         stubs.api.container_requests().create.assert_called_with(
1517             body=JsonDiffMatcher(expect_container))
1518         self.assertEqual(stubs.capture_stdout.getvalue(),
1519                          stubs.expect_container_request_uuid + '\n')
1520         self.assertEqual(exited, 0)
1521
1522
1523 class TestCreateWorkflow(unittest.TestCase):
1524     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1525     expect_workflow = StripYAMLComments(
1526         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1527     expect_workflow_altname = StripYAMLComments(
1528         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1529
1530     def setUp(self):
1531         cwltool.process._names = set()
1532         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1533
1534     def tearDown(self):
1535         root_logger = logging.getLogger('')
1536
1537         # Remove existing RuntimeStatusLoggingHandlers if they exist
1538         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1539         root_logger.handlers = handlers
1540
1541     @stubs()
1542     def test_create(self, stubs):
1543         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1544         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1545
1546         exited = arvados_cwl.main(
1547             ["--create-workflow", "--debug",
1548              "--api=containers",
1549              "--project-uuid", project_uuid,
1550              "--disable-git",
1551              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1552             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1553
1554         stubs.api.pipeline_templates().create.refute_called()
1555         stubs.api.container_requests().create.refute_called()
1556
1557         body = {
1558             "workflow": {
1559                 "owner_uuid": project_uuid,
1560                 "name": "submit_wf.cwl",
1561                 "description": "",
1562                 "definition": self.expect_workflow,
1563             }
1564         }
1565         stubs.api.workflows().create.assert_called_with(
1566             body=JsonDiffMatcher(body))
1567
1568         self.assertEqual(stubs.capture_stdout.getvalue(),
1569                          stubs.expect_workflow_uuid + '\n')
1570         self.assertEqual(exited, 0)
1571
1572     @stubs()
1573     def test_create_name(self, stubs):
1574         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1575         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1576
1577         exited = arvados_cwl.main(
1578             ["--create-workflow", "--debug",
1579              "--api=containers",
1580              "--project-uuid", project_uuid,
1581              "--name", "testing 123",
1582              "--disable-git",
1583              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1584             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1585
1586         stubs.api.pipeline_templates().create.refute_called()
1587         stubs.api.container_requests().create.refute_called()
1588
1589         body = {
1590             "workflow": {
1591                 "owner_uuid": project_uuid,
1592                 "name": "testing 123",
1593                 "description": "",
1594                 "definition": self.expect_workflow_altname,
1595             }
1596         }
1597         stubs.api.workflows().create.assert_called_with(
1598             body=JsonDiffMatcher(body))
1599
1600         self.assertEqual(stubs.capture_stdout.getvalue(),
1601                          stubs.expect_workflow_uuid + '\n')
1602         self.assertEqual(exited, 0)
1603
1604
1605     @stubs()
1606     def test_update(self, stubs):
1607         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1608         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1609
1610         exited = arvados_cwl.main(
1611             ["--update-workflow", self.existing_workflow_uuid,
1612              "--debug",
1613              "--disable-git",
1614              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1615             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1616
1617         body = {
1618             "workflow": {
1619                 "name": "submit_wf.cwl",
1620                 "description": "",
1621                 "definition": self.expect_workflow,
1622                 "owner_uuid": project_uuid
1623             }
1624         }
1625         stubs.api.workflows().update.assert_called_with(
1626             uuid=self.existing_workflow_uuid,
1627             body=JsonDiffMatcher(body))
1628         self.assertEqual(stubs.capture_stdout.getvalue(),
1629                          self.existing_workflow_uuid + '\n')
1630         self.assertEqual(exited, 0)
1631
1632
1633     @stubs()
1634     def test_update_name(self, stubs):
1635         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1636         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1637
1638         exited = arvados_cwl.main(
1639             ["--update-workflow", self.existing_workflow_uuid,
1640              "--debug", "--name", "testing 123",
1641              "--disable-git",
1642              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1643             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1644
1645         body = {
1646             "workflow": {
1647                 "name": "testing 123",
1648                 "description": "",
1649                 "definition": self.expect_workflow_altname,
1650                 "owner_uuid": project_uuid
1651             }
1652         }
1653         stubs.api.workflows().update.assert_called_with(
1654             uuid=self.existing_workflow_uuid,
1655             body=JsonDiffMatcher(body))
1656         self.assertEqual(stubs.capture_stdout.getvalue(),
1657                          self.existing_workflow_uuid + '\n')
1658         self.assertEqual(exited, 0)
1659
1660     @stubs()
1661     def test_create_collection_per_tool(self, stubs):
1662         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1663         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1664
1665         exited = arvados_cwl.main(
1666             ["--create-workflow", "--debug",
1667              "--api=containers",
1668              "--project-uuid", project_uuid,
1669              "--disable-git",
1670              "tests/collection_per_tool/collection_per_tool.cwl"],
1671             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1672
1673         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1674         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1675
1676         body = {
1677             "workflow": {
1678                 "owner_uuid": project_uuid,
1679                 "name": "collection_per_tool.cwl",
1680                 "description": "",
1681                 "definition": expect_workflow,
1682             }
1683         }
1684         stubs.api.workflows().create.assert_called_with(
1685             body=JsonDiffMatcher(body))
1686
1687         self.assertEqual(stubs.capture_stdout.getvalue(),
1688                          stubs.expect_workflow_uuid + '\n')
1689         self.assertEqual(exited, 0)
1690
1691     @stubs()
1692     def test_create_with_imports(self, stubs):
1693         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1694         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1695
1696         exited = arvados_cwl.main(
1697             ["--create-workflow", "--debug",
1698              "--api=containers",
1699              "--project-uuid", project_uuid,
1700              "tests/wf/feddemo/feddemo.cwl"],
1701             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1702
1703         stubs.api.pipeline_templates().create.refute_called()
1704         stubs.api.container_requests().create.refute_called()
1705
1706         self.assertEqual(stubs.capture_stdout.getvalue(),
1707                          stubs.expect_workflow_uuid + '\n')
1708         self.assertEqual(exited, 0)
1709
1710     @stubs()
1711     def test_create_with_no_input(self, stubs):
1712         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1713         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1714
1715         exited = arvados_cwl.main(
1716             ["--create-workflow", "--debug",
1717              "--api=containers",
1718              "--project-uuid", project_uuid,
1719              "tests/wf/revsort/revsort.cwl"],
1720             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1721
1722         stubs.api.pipeline_templates().create.refute_called()
1723         stubs.api.container_requests().create.refute_called()
1724
1725         self.assertEqual(stubs.capture_stdout.getvalue(),
1726                          stubs.expect_workflow_uuid + '\n')
1727         self.assertEqual(exited, 0)
1728
1729     @stubs()
1730     def test_create_map(self, stubs):
1731         # test uploading a document that uses objects instead of arrays
1732         # for certain fields like inputs and requirements.
1733
1734         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1735         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1736
1737         exited = arvados_cwl.main(
1738             ["--create-workflow", "--debug",
1739              "--api=containers",
1740              "--project-uuid", project_uuid,
1741              "--disable-git",
1742              "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
1743             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1744
1745         stubs.api.pipeline_templates().create.refute_called()
1746         stubs.api.container_requests().create.refute_called()
1747
1748         expect_workflow = StripYAMLComments(
1749             open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
1750
1751         body = {
1752             "workflow": {
1753                 "owner_uuid": project_uuid,
1754                 "name": "submit_wf_map.cwl",
1755                 "description": "",
1756                 "definition": expect_workflow,
1757             }
1758         }
1759         stubs.api.workflows().create.assert_called_with(
1760             body=JsonDiffMatcher(body))
1761
1762         self.assertEqual(stubs.capture_stdout.getvalue(),
1763                          stubs.expect_workflow_uuid + '\n')
1764         self.assertEqual(exited, 0)
1765
1766
1767 class TestPrintKeepDeps(unittest.TestCase):
1768     @stubs()
1769     def test_print_keep_deps(self, stubs):
1770         # test --print-keep-deps which is used by arv-copy
1771
1772         exited = arvados_cwl.main(
1773             ["--print-keep-deps", "--debug",
1774              "tests/wf/submit_wf_map.cwl"],
1775             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1776
1777         self.assertEqual(stubs.capture_stdout.getvalue(),
1778                          '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
1779         self.assertEqual(exited, 0)