sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=0',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343         arvados_cwl.arvdocker.arv_docker_clear_cache()
 344
 345     @stubs
 346     def test_error_when_multiple_storage_classes_specified(self, stubs):
 347         storage_classes = "foo,bar"
 348         exited = arvados_cwl.main(
 349                 ["--debug", "--storage-classes", storage_classes,
 350                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351                 sys.stdin, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 1)
 353
 354     @mock.patch("time.sleep")
 355     @stubs
 356     def test_submit_invalid_runner_ram(self, stubs, tm):
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 361         self.assertEqual(exited, 1)
 362
 363
 364     @stubs
 365     def test_submit_container(self, stubs):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--api=containers", "--debug",
 368                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 370
 371         stubs.api.collections().create.assert_has_calls([
 372             mock.call(body=JsonDiffMatcher({
 373                 'manifest_text':
 374                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 375                 'replication_desired': None,
 376                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 377             }), ensure_unique_name=False),
 378             mock.call(body=JsonDiffMatcher({
 379                 'manifest_text':
 380                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 381                 'replication_desired': None,
 382                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 383             }), ensure_unique_name=False),
 384             ])
 385
 386         expect_container = copy.deepcopy(stubs.expect_container_spec)
 387         stubs.api.container_requests().create.assert_called_with(
 388             body=JsonDiffMatcher(expect_container))
 389         self.assertEqual(stubs.capture_stdout.getvalue(),
 390                          stubs.expect_container_request_uuid + '\n')
 391         self.assertEqual(exited, 0)
 392
 393
 394     @stubs
 395     def test_submit_container_tool(self, stubs):
 396         # test for issue #16139
 397         exited = arvados_cwl.main(
 398             ["--submit", "--no-wait", "--api=containers", "--debug",
 399                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 400             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 401
 402         self.assertEqual(stubs.capture_stdout.getvalue(),
 403                          stubs.expect_container_request_uuid + '\n')
 404         self.assertEqual(exited, 0)
 405
 406     @stubs
 407     def test_submit_container_no_reuse(self, stubs):
 408         exited = arvados_cwl.main(
 409             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 410                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         expect_container["command"] = [
 415             'arvados-cwl-runner', '--local', '--api=containers',
 416             '--no-log-timestamps', '--disable-validate', '--disable-color',
 417             '--eval-timeout=20', '--thread-count=0',
 418             '--disable-reuse', "--collection-cache-size=256",
 419             '--debug', '--on-error=continue',
 420             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 421         expect_container["use_existing"] = False
 422
 423         stubs.api.container_requests().create.assert_called_with(
 424             body=JsonDiffMatcher(expect_container))
 425         self.assertEqual(stubs.capture_stdout.getvalue(),
 426                          stubs.expect_container_request_uuid + '\n')
 427         self.assertEqual(exited, 0)
 428
 429     @stubs
 430     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug",
 433              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435         self.assertEqual(exited, 0)
 436
 437         expect_container = copy.deepcopy(stubs.expect_container_spec)
 438         expect_container["command"] = [
 439             'arvados-cwl-runner', '--local', '--api=containers',
 440             '--no-log-timestamps', '--disable-validate', '--disable-color',
 441             '--eval-timeout=20', '--thread-count=0',
 442             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 443             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 444         expect_container["use_existing"] = False
 445         expect_container["name"] = "submit_wf_no_reuse.cwl"
 446         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 447             {
 448                 "class": "http://arvados.org/cwl#ReuseRequirement",
 449                 "enableReuse": False,
 450             },
 451         ]
 452         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 453             "arv": "http://arvados.org/cwl#",
 454             "cwltool": "http://commonwl.org/cwltool#"
 455         }
 456
 457         stubs.api.container_requests().create.assert_called_with(
 458             body=JsonDiffMatcher(expect_container))
 459         self.assertEqual(stubs.capture_stdout.getvalue(),
 460                          stubs.expect_container_request_uuid + '\n')
 461
 462
 463     @stubs
 464     def test_submit_container_on_error(self, stubs):
 465         exited = arvados_cwl.main(
 466             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 467                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 468             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 469
 470         expect_container = copy.deepcopy(stubs.expect_container_spec)
 471         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 472                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 473                                        '--eval-timeout=20', '--thread-count=0',
 474                                        '--enable-reuse', "--collection-cache-size=256",
 475                                        '--debug', '--on-error=stop',
 476                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 477
 478         stubs.api.container_requests().create.assert_called_with(
 479             body=JsonDiffMatcher(expect_container))
 480         self.assertEqual(stubs.capture_stdout.getvalue(),
 481                          stubs.expect_container_request_uuid + '\n')
 482         self.assertEqual(exited, 0)
 483
 484     @stubs
 485     def test_submit_container_output_name(self, stubs):
 486         output_name = "test_output_name"
 487
 488         exited = arvados_cwl.main(
 489             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 490                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 491             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 492
 493         expect_container = copy.deepcopy(stubs.expect_container_spec)
 494         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 495                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 496                                        '--eval-timeout=20', '--thread-count=0',
 497                                        '--enable-reuse', "--collection-cache-size=256",
 498                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 499                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 500         expect_container["output_name"] = output_name
 501
 502         stubs.api.container_requests().create.assert_called_with(
 503             body=JsonDiffMatcher(expect_container))
 504         self.assertEqual(stubs.capture_stdout.getvalue(),
 505                          stubs.expect_container_request_uuid + '\n')
 506         self.assertEqual(exited, 0)
 507
 508     @stubs
 509     def test_submit_storage_classes(self, stubs):
 510         exited = arvados_cwl.main(
 511             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 512                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514
 515         expect_container = copy.deepcopy(stubs.expect_container_spec)
 516         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 517                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 518                                        '--eval-timeout=20', '--thread-count=0',
 519                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 520                                        "--storage-classes=foo", '--on-error=continue',
 521                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 522
 523         stubs.api.container_requests().create.assert_called_with(
 524             body=JsonDiffMatcher(expect_container))
 525         self.assertEqual(stubs.capture_stdout.getvalue(),
 526                          stubs.expect_container_request_uuid + '\n')
 527         self.assertEqual(exited, 0)
 528
 529     @mock.patch("cwltool.task_queue.TaskQueue")
 530     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 531     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 532     @stubs
 533     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 534         final_output_c = arvados.collection.Collection()
 535         make_output.return_value = ({},final_output_c)
 536
 537         def set_final_output(job_order, output_callback, runtimeContext):
 538             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 539             return []
 540         job.side_effect = set_final_output
 541
 542         exited = arvados_cwl.main(
 543             ["--debug", "--local", "--storage-classes=foo",
 544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 545             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 546
 547         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 548         self.assertEqual(exited, 0)
 549
 550     @mock.patch("cwltool.task_queue.TaskQueue")
 551     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 552     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 553     @stubs
 554     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 555         final_output_c = arvados.collection.Collection()
 556         make_output.return_value = ({},final_output_c)
 557
 558         def set_final_output(job_order, output_callback, runtimeContext):
 559             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 560             return []
 561         job.side_effect = set_final_output
 562
 563         exited = arvados_cwl.main(
 564             ["--debug", "--local",
 565                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 566             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 567
 568         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 569         self.assertEqual(exited, 0)
 570
 571     @stubs
 572     def test_submit_container_output_ttl(self, stubs):
 573         exited = arvados_cwl.main(
 574             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 575                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 576             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 577
 578         expect_container = copy.deepcopy(stubs.expect_container_spec)
 579         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 580                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 581                                        '--eval-timeout=20', '--thread-count=0',
 582                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 583                                        '--on-error=continue',
 584                                        "--intermediate-output-ttl=3600",
 585                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586
 587         stubs.api.container_requests().create.assert_called_with(
 588             body=JsonDiffMatcher(expect_container))
 589         self.assertEqual(stubs.capture_stdout.getvalue(),
 590                          stubs.expect_container_request_uuid + '\n')
 591         self.assertEqual(exited, 0)
 592
 593     @stubs
 594     def test_submit_container_trash_intermediate(self, stubs):
 595         exited = arvados_cwl.main(
 596             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 597                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599
 600
 601         expect_container = copy.deepcopy(stubs.expect_container_spec)
 602         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 603                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 604                                        '--eval-timeout=20', '--thread-count=0',
 605                                        '--enable-reuse', "--collection-cache-size=256",
 606                                        '--debug', '--on-error=continue',
 607                                        "--trash-intermediate",
 608                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 609
 610         stubs.api.container_requests().create.assert_called_with(
 611             body=JsonDiffMatcher(expect_container))
 612         self.assertEqual(stubs.capture_stdout.getvalue(),
 613                          stubs.expect_container_request_uuid + '\n')
 614         self.assertEqual(exited, 0)
 615
 616     @stubs
 617     def test_submit_container_output_tags(self, stubs):
 618         output_tags = "tag0,tag1,tag2"
 619
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 627                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 628                                        '--eval-timeout=20', '--thread-count=0',
 629                                        '--enable-reuse', "--collection-cache-size=256",
 630                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 631                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 632
 633         stubs.api.container_requests().create.assert_called_with(
 634             body=JsonDiffMatcher(expect_container))
 635         self.assertEqual(stubs.capture_stdout.getvalue(),
 636                          stubs.expect_container_request_uuid + '\n')
 637         self.assertEqual(exited, 0)
 638
 639     @stubs
 640     def test_submit_container_runner_ram(self, stubs):
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 643                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 645
 646         expect_container = copy.deepcopy(stubs.expect_container_spec)
 647         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(stubs.capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653         self.assertEqual(exited, 0)
 654
 655     @mock.patch("arvados.collection.CollectionReader")
 656     @mock.patch("time.sleep")
 657     @stubs
 658     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 659         collectionReader().exists.return_value = True
 660         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug",
 663              "tests/wf/submit_keepref_wf.cwl"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 665         self.assertEqual(exited, 0)
 666
 667     @mock.patch("arvados.collection.CollectionReader")
 668     @mock.patch("time.sleep")
 669     @stubs
 670     def test_submit_keepref(self, stubs, tm, reader):
 671         with open("tests/wf/expect_arvworkflow.cwl") as f:
 672             reader().open().__enter__().read.return_value = f.read()
 673
 674         exited = arvados_cwl.main(
 675             ["--submit", "--no-wait", "--api=containers", "--debug",
 676              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 677             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 678
 679         expect_container = {
 680             'priority': 500,
 681             'mounts': {
 682                 '/var/spool/cwl': {
 683                     'writable': True,
 684                     'kind': 'collection'
 685                 },
 686                 'stdout': {
 687                     'path': '/var/spool/cwl/cwl.output.json',
 688                     'kind': 'file'
 689                 },
 690                 '/var/lib/cwl/workflow': {
 691                     'portable_data_hash': '99999999999999999999999999999994+99',
 692                     'kind': 'collection'
 693                 },
 694                 '/var/lib/cwl/cwl.input.json': {
 695                     'content': {
 696                         'x': 'XxX'
 697                     },
 698                     'kind': 'json'
 699                 }
 700             }, 'state': 'Committed',
 701             'output_path': '/var/spool/cwl',
 702             'name': 'expect_arvworkflow.cwl#main',
 703             'container_image': '999999999999999999999999999999d3+99',
 704             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 705                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 706                         '--eval-timeout=20', '--thread-count=0',
 707                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 708                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 709             'cwd': '/var/spool/cwl',
 710             'runtime_constraints': {
 711                 'API': True,
 712                 'vcpus': 1,
 713                 'ram': 1342177280
 714             },
 715             'use_existing': False,
 716             'properties': {},
 717             'secret_mounts': {}
 718         }
 719
 720         stubs.api.container_requests().create.assert_called_with(
 721             body=JsonDiffMatcher(expect_container))
 722         self.assertEqual(stubs.capture_stdout.getvalue(),
 723                          stubs.expect_container_request_uuid + '\n')
 724         self.assertEqual(exited, 0)
 725
 726     @mock.patch("time.sleep")
 727     @stubs
 728     def test_submit_arvworkflow(self, stubs, tm):
 729         with open("tests/wf/expect_arvworkflow.cwl") as f:
 730             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 731
 732         exited = arvados_cwl.main(
 733             ["--submit", "--no-wait", "--api=containers", "--debug",
 734              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 735             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 736
 737         expect_container = {
 738             'priority': 500,
 739             'mounts': {
 740                 '/var/spool/cwl': {
 741                     'writable': True,
 742                     'kind': 'collection'
 743                 },
 744                 'stdout': {
 745                     'path': '/var/spool/cwl/cwl.output.json',
 746                     'kind': 'file'
 747                 },
 748                 '/var/lib/cwl/workflow.json': {
 749                     'kind': 'json',
 750                     'content': {
 751                         'cwlVersion': 'v1.0',
 752                         '$graph': [
 753                             {
 754                                 'id': '#main',
 755                                 'inputs': [
 756                                     {'type': 'string', 'id': '#main/x'}
 757                                 ],
 758                                 'steps': [
 759                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 760                                      'run': '#submit_tool.cwl',
 761                                      'id': '#main/step1',
 762                                      'out': []}
 763                                 ],
 764                                 'class': 'Workflow',
 765                                 'outputs': []
 766                             },
 767                             {
 768                                 'inputs': [
 769                                     {
 770                                         'inputBinding': {'position': 1},
 771                                         'type': 'string',
 772                                         'id': '#submit_tool.cwl/x'}
 773                                 ],
 774                                 'requirements': [
 775                                     {
 776                                         'dockerPull': 'debian:buster-slim',
 777                                         'class': 'DockerRequirement',
 778                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 779                                     }
 780                                 ],
 781                                 'id': '#submit_tool.cwl',
 782                                 'outputs': [],
 783                                 'baseCommand': 'cat',
 784                                 'class': 'CommandLineTool'
 785                             }
 786                         ]
 787                     }
 788                 },
 789                 '/var/lib/cwl/cwl.input.json': {
 790                     'content': {
 791                         'x': 'XxX'
 792                     },
 793                     'kind': 'json'
 794                 }
 795             }, 'state': 'Committed',
 796             'output_path': '/var/spool/cwl',
 797             'name': 'a test workflow',
 798             'container_image': "999999999999999999999999999999d3+99",
 799             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 800                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 801                         '--eval-timeout=20', '--thread-count=0',
 802                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 803                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 804             'cwd': '/var/spool/cwl',
 805             'runtime_constraints': {
 806                 'API': True,
 807                 'vcpus': 1,
 808                 'ram': 1342177280
 809             },
 810             'use_existing': False,
 811             'properties': {
 812                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 813             },
 814             'secret_mounts': {}
 815         }
 816
 817         stubs.api.container_requests().create.assert_called_with(
 818             body=JsonDiffMatcher(expect_container))
 819         self.assertEqual(stubs.capture_stdout.getvalue(),
 820                          stubs.expect_container_request_uuid + '\n')
 821         self.assertEqual(exited, 0)
 822
 823     @stubs
 824     def test_submit_container_name(self, stubs):
 825         exited = arvados_cwl.main(
 826             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 827                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 828             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 829
 830         expect_container = copy.deepcopy(stubs.expect_container_spec)
 831         expect_container["name"] = "hello container 123"
 832
 833         stubs.api.container_requests().create.assert_called_with(
 834             body=JsonDiffMatcher(expect_container))
 835         self.assertEqual(stubs.capture_stdout.getvalue(),
 836                          stubs.expect_container_request_uuid + '\n')
 837         self.assertEqual(exited, 0)
 838
 839     @stubs
 840     def test_submit_missing_input(self, stubs):
 841         exited = arvados_cwl.main(
 842             ["--submit", "--no-wait", "--api=containers", "--debug",
 843              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 844             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 845         self.assertEqual(exited, 0)
 846
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug",
 849              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851         self.assertEqual(exited, 1)
 852
 853     @stubs
 854     def test_submit_container_project(self, stubs):
 855         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 856         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 857         exited = arvados_cwl.main(
 858             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 859                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 860             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 861
 862         expect_container = copy.deepcopy(stubs.expect_container_spec)
 863         expect_container["owner_uuid"] = project_uuid
 864         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 865                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 866                                        "--eval-timeout=20", "--thread-count=0",
 867                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 868                                        '--on-error=continue',
 869                                        '--project-uuid='+project_uuid,
 870                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 871
 872         stubs.api.container_requests().create.assert_called_with(
 873             body=JsonDiffMatcher(expect_container))
 874         self.assertEqual(stubs.capture_stdout.getvalue(),
 875                          stubs.expect_container_request_uuid + '\n')
 876         self.assertEqual(exited, 0)
 877
 878     @stubs
 879     def test_submit_container_eval_timeout(self, stubs):
 880         exited = arvados_cwl.main(
 881             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 882                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 883             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 884
 885         expect_container = copy.deepcopy(stubs.expect_container_spec)
 886         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 887                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 888                                        '--eval-timeout=60.0', '--thread-count=0',
 889                                        '--enable-reuse', "--collection-cache-size=256",
 890                                        '--debug', '--on-error=continue',
 891                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 892
 893         stubs.api.container_requests().create.assert_called_with(
 894             body=JsonDiffMatcher(expect_container))
 895         self.assertEqual(stubs.capture_stdout.getvalue(),
 896                          stubs.expect_container_request_uuid + '\n')
 897         self.assertEqual(exited, 0)
 898
 899     @stubs
 900     def test_submit_container_collection_cache(self, stubs):
 901         exited = arvados_cwl.main(
 902             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 903                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 904             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 905
 906         expect_container = copy.deepcopy(stubs.expect_container_spec)
 907         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 908                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 909                                        '--eval-timeout=20', '--thread-count=0',
 910                                        '--enable-reuse', "--collection-cache-size=500",
 911                                        '--debug', '--on-error=continue',
 912                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 913         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 914
 915         stubs.api.container_requests().create.assert_called_with(
 916             body=JsonDiffMatcher(expect_container))
 917         self.assertEqual(stubs.capture_stdout.getvalue(),
 918                          stubs.expect_container_request_uuid + '\n')
 919         self.assertEqual(exited, 0)
 920
 921     @stubs
 922     def test_submit_container_thread_count(self, stubs):
 923         exited = arvados_cwl.main(
 924             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 925                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 926             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 927
 928         expect_container = copy.deepcopy(stubs.expect_container_spec)
 929         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 930                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 931                                        '--eval-timeout=20', '--thread-count=20',
 932                                        '--enable-reuse', "--collection-cache-size=256",
 933                                        '--debug', '--on-error=continue',
 934                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 935
 936         stubs.api.container_requests().create.assert_called_with(
 937             body=JsonDiffMatcher(expect_container))
 938         self.assertEqual(stubs.capture_stdout.getvalue(),
 939                          stubs.expect_container_request_uuid + '\n')
 940         self.assertEqual(exited, 0)
 941
 942     @stubs
 943     def test_submit_container_runner_image(self, stubs):
 944         exited = arvados_cwl.main(
 945             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 946                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 947             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 948
 949         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 950
 951         expect_container = copy.deepcopy(stubs.expect_container_spec)
 952         stubs.api.container_requests().create.assert_called_with(
 953             body=JsonDiffMatcher(expect_container))
 954         self.assertEqual(stubs.capture_stdout.getvalue(),
 955                          stubs.expect_container_request_uuid + '\n')
 956         self.assertEqual(exited, 0)
 957
 958     @stubs
 959     def test_submit_priority(self, stubs):
 960         exited = arvados_cwl.main(
 961             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 962                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 963             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 964
 965         stubs.expect_container_spec["priority"] = 669
 966
 967         expect_container = copy.deepcopy(stubs.expect_container_spec)
 968         stubs.api.container_requests().create.assert_called_with(
 969             body=JsonDiffMatcher(expect_container))
 970         self.assertEqual(stubs.capture_stdout.getvalue(),
 971                          stubs.expect_container_request_uuid + '\n')
 972         self.assertEqual(exited, 0)
 973
 974     @stubs
 975     def test_submit_wf_runner_resources(self, stubs):
 976         exited = arvados_cwl.main(
 977             ["--submit", "--no-wait", "--api=containers", "--debug",
 978                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 979             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 980
 981         expect_container = copy.deepcopy(stubs.expect_container_spec)
 982         expect_container["runtime_constraints"] = {
 983             "API": True,
 984             "vcpus": 2,
 985             "ram": (2000+512) * 2**20
 986         }
 987         expect_container["name"] = "submit_wf_runner_resources.cwl"
 988         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 989             {
 990                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 991                 "coresMin": 2,
 992                 "ramMin": 2000,
 993                 "keep_cache": 512
 994             }
 995         ]
 996         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 997             "arv": "http://arvados.org/cwl#",
 998         }
 999         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1000                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1001                         '--eval-timeout=20', '--thread-count=0',
1002                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1003                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1004
1005         stubs.api.container_requests().create.assert_called_with(
1006             body=JsonDiffMatcher(expect_container))
1007         self.assertEqual(stubs.capture_stdout.getvalue(),
1008                          stubs.expect_container_request_uuid + '\n')
1009         self.assertEqual(exited, 0)
1010
1011     def tearDown(self):
1012         arvados_cwl.arvdocker.arv_docker_clear_cache()
1013
1014     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1015     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1016     @mock.patch("arvados.api")
1017     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1018         arvados_cwl.arvdocker.arv_docker_clear_cache()
1019
1020         arvrunner = mock.MagicMock()
1021         arvrunner.project_uuid = ""
1022         api.return_value = mock.MagicMock()
1023         arvrunner.api = api.return_value
1024         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1025                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1026                                                                         "link_class": "docker_image_repo+tag",
1027                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1028                                                                         "owner_uuid": "",
1029                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1030                                                             {"items": [{"created_at": "",
1031                                                                         "head_uuid": "",
1032                                                                         "link_class": "docker_image_hash",
1033                                                                         "name": "123456",
1034                                                                         "owner_uuid": "",
1035                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1036         )
1037         find_one_image_hash.return_value = "123456"
1038
1039         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1040                                                                               "owner_uuid": "",
1041                                                                               "manifest_text": "",
1042                                                                               "properties": ""
1043                                                                           }], "items_available": 1, "offset": 0},)
1044         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1045         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1046                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1047         self.assertEqual("9999999999999999999999999999999b+99",
1048                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1049
1050
1051     @stubs
1052     def test_submit_secrets(self, stubs):
1053         exited = arvados_cwl.main(
1054             ["--submit", "--no-wait", "--api=containers", "--debug",
1055                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1056             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1057
1058         expect_container = {
1059             "command": [
1060                 "arvados-cwl-runner",
1061                 "--local",
1062                 "--api=containers",
1063                 "--no-log-timestamps",
1064                 "--disable-validate",
1065                 "--disable-color",
1066                 "--eval-timeout=20",
1067                 '--thread-count=0',
1068                 "--enable-reuse",
1069                 "--collection-cache-size=256",
1070                 '--debug',
1071                 "--on-error=continue",
1072                 "/var/lib/cwl/workflow.json#main",
1073                 "/var/lib/cwl/cwl.input.json"
1074             ],
1075             "container_image": "999999999999999999999999999999d3+99",
1076             "cwd": "/var/spool/cwl",
1077             "mounts": {
1078                 "/var/lib/cwl/cwl.input.json": {
1079                     "content": {
1080                         "pw": {
1081                             "$include": "/secrets/s0"
1082                         }
1083                     },
1084                     "kind": "json"
1085                 },
1086                 "/var/lib/cwl/workflow.json": {
1087                     "content": {
1088                         "$graph": [
1089                             {
1090                                 "$namespaces": {
1091                                     "cwltool": "http://commonwl.org/cwltool#"
1092                                 },
1093                                 "arguments": [
1094                                     "md5sum",
1095                                     "example.conf"
1096                                 ],
1097                                 "class": "CommandLineTool",
1098                                 "hints": [
1099                                     {
1100                                         "class": "http://commonwl.org/cwltool#Secrets",
1101                                         "secrets": [
1102                                             "#secret_job.cwl/pw"
1103                                         ]
1104                                     }
1105                                 ],
1106                                 "id": "#secret_job.cwl",
1107                                 "inputs": [
1108                                     {
1109                                         "id": "#secret_job.cwl/pw",
1110                                         "type": "string"
1111                                     }
1112                                 ],
1113                                 "outputs": [
1114                                     {
1115                                         "id": "#secret_job.cwl/out",
1116                                         "type": "File",
1117                                         "outputBinding": {
1118                                               "glob": "hashed_example.txt"
1119                                         }
1120                                     }
1121                                 ],
1122                                 "stdout": "hashed_example.txt",
1123                                 "requirements": [
1124                                     {
1125                                         "class": "InitialWorkDirRequirement",
1126                                         "listing": [
1127                                             {
1128                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1129                                                 "entryname": "example.conf"
1130                                             }
1131                                         ]
1132                                     }
1133                                 ]
1134                             },
1135                             {
1136                                 "class": "Workflow",
1137                                 "hints": [
1138                                     {
1139                                         "class": "DockerRequirement",
1140                                         "dockerPull": "debian:buster-slim",
1141                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1142                                     },
1143                                     {
1144                                         "class": "http://commonwl.org/cwltool#Secrets",
1145                                         "secrets": [
1146                                             "#main/pw"
1147                                         ]
1148                                     }
1149                                 ],
1150                                 "id": "#main",
1151                                 "inputs": [
1152                                     {
1153                                         "id": "#main/pw",
1154                                         "type": "string"
1155                                     }
1156                                 ],
1157                                 "outputs": [
1158                                     {
1159                                         "id": "#main/out",
1160                                         "outputSource": "#main/step1/out",
1161                                         "type": "File"
1162                                     }
1163                                 ],
1164                                 "steps": [
1165                                     {
1166                                         "id": "#main/step1",
1167                                         "in": [
1168                                             {
1169                                                 "id": "#main/step1/pw",
1170                                                 "source": "#main/pw"
1171                                             }
1172                                         ],
1173                                         "out": [
1174                                             "#main/step1/out"
1175                                         ],
1176                                         "run": "#secret_job.cwl"
1177                                     }
1178                                 ]
1179                             }
1180                         ],
1181                         "cwlVersion": "v1.0"
1182                     },
1183                     "kind": "json"
1184                 },
1185                 "/var/spool/cwl": {
1186                     "kind": "collection",
1187                     "writable": True
1188                 },
1189                 "stdout": {
1190                     "kind": "file",
1191                     "path": "/var/spool/cwl/cwl.output.json"
1192                 }
1193             },
1194             "name": "secret_wf.cwl",
1195             "output_path": "/var/spool/cwl",
1196             "priority": 500,
1197             "properties": {},
1198             "runtime_constraints": {
1199                 "API": True,
1200                 "ram": 1342177280,
1201                 "vcpus": 1
1202             },
1203             "secret_mounts": {
1204                 "/secrets/s0": {
1205                     "content": "blorp",
1206                     "kind": "text"
1207                 }
1208             },
1209             "state": "Committed",
1210             "use_existing": False
1211         }
1212
1213         stubs.api.container_requests().create.assert_called_with(
1214             body=JsonDiffMatcher(expect_container))
1215         self.assertEqual(stubs.capture_stdout.getvalue(),
1216                          stubs.expect_container_request_uuid + '\n')
1217         self.assertEqual(exited, 0)
1218
1219     @stubs
1220     def test_submit_request_uuid(self, stubs):
1221         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1222         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1223
1224         stubs.api.container_requests().update().execute.return_value = {
1225             "uuid": stubs.expect_container_request_uuid,
1226             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1227             "state": "Queued"
1228         }
1229
1230         exited = arvados_cwl.main(
1231             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1232                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1233             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1234
1235         stubs.api.container_requests().update.assert_called_with(
1236             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1237         self.assertEqual(stubs.capture_stdout.getvalue(),
1238                          stubs.expect_container_request_uuid + '\n')
1239         self.assertEqual(exited, 0)
1240
1241     @stubs
1242     def test_submit_container_cluster_id(self, stubs):
1243         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1244
1245         exited = arvados_cwl.main(
1246             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1247                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1248             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1249
1250         expect_container = copy.deepcopy(stubs.expect_container_spec)
1251
1252         stubs.api.container_requests().create.assert_called_with(
1253             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1254         self.assertEqual(stubs.capture_stdout.getvalue(),
1255                          stubs.expect_container_request_uuid + '\n')
1256         self.assertEqual(exited, 0)
1257
1258     @stubs
1259     def test_submit_validate_cluster_id(self, stubs):
1260         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1261         exited = arvados_cwl.main(
1262             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1263              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1264             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1265         self.assertEqual(exited, 1)
1266
1267     @stubs
1268     def test_submit_validate_project_uuid(self, stubs):
1269         # Fails with bad cluster prefix
1270         exited = arvados_cwl.main(
1271             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1272              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1273             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1274         self.assertEqual(exited, 1)
1275
1276         # Project lookup fails
1277         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1278         exited = arvados_cwl.main(
1279             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1281             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1282         self.assertEqual(exited, 1)
1283
1284         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1285         exited = arvados_cwl.main(
1286             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1287              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1288             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1289         self.assertEqual(exited, 0)
1290
1291
1292     @mock.patch("arvados.collection.CollectionReader")
1293     @stubs
1294     def test_submit_uuid_inputs(self, stubs, collectionReader):
1295         collectionReader().exists.return_value = True
1296         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1297         def list_side_effect(**kwargs):
1298             m = mock.MagicMock()
1299             if "count" in kwargs:
1300                 m.execute.return_value = {"items": [
1301                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1302                 ]}
1303             else:
1304                 m.execute.return_value = {"items": []}
1305             return m
1306         stubs.api.collections().list.side_effect = list_side_effect
1307
1308         exited = arvados_cwl.main(
1309             ["--submit", "--no-wait", "--api=containers", "--debug",
1310                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1311             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1312
1313         expect_container = copy.deepcopy(stubs.expect_container_spec)
1314         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1315         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1316         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1317
1318         stubs.api.collections().list.assert_has_calls([
1319             mock.call(count='none',
1320                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1321                       select=['uuid', 'portable_data_hash'])])
1322         stubs.api.container_requests().create.assert_called_with(
1323             body=JsonDiffMatcher(expect_container))
1324         self.assertEqual(stubs.capture_stdout.getvalue(),
1325                          stubs.expect_container_request_uuid + '\n')
1326         self.assertEqual(exited, 0)
1327
1328     @stubs
1329     def test_submit_mismatched_uuid_inputs(self, stubs):
1330         def list_side_effect(**kwargs):
1331             m = mock.MagicMock()
1332             if "count" in kwargs:
1333                 m.execute.return_value = {"items": [
1334                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1335                 ]}
1336             else:
1337                 m.execute.return_value = {"items": []}
1338             return m
1339         stubs.api.collections().list.side_effect = list_side_effect
1340
1341         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1342             capture_stderr = StringIO()
1343             cwltool_logger = logging.getLogger('cwltool')
1344             stderr_logger = logging.StreamHandler(capture_stderr)
1345             cwltool_logger.addHandler(stderr_logger)
1346
1347             try:
1348                 exited = arvados_cwl.main(
1349                     ["--submit", "--no-wait", "--api=containers", "--debug",
1350                         "tests/wf/submit_wf.cwl", infile],
1351                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1352
1353                 self.assertEqual(exited, 1)
1354                 self.assertRegex(
1355                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1356                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1357             finally:
1358                 cwltool_logger.removeHandler(stderr_logger)
1359
1360     @mock.patch("arvados.collection.CollectionReader")
1361     @stubs
1362     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1363         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1364         capture_stderr = StringIO()
1365
1366         cwltool_logger = logging.getLogger('cwltool')
1367         stderr_logger = logging.StreamHandler(capture_stderr)
1368         cwltool_logger.addHandler(stderr_logger)
1369
1370         exited = arvados_cwl.main(
1371             ["--submit", "--no-wait", "--api=containers", "--debug",
1372                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1373             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1374
1375         try:
1376             self.assertEqual(exited, 1)
1377             self.assertRegex(
1378                 capture_stderr.getvalue(),
1379                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1380         finally:
1381             cwltool_logger.removeHandler(stderr_logger)
1382
1383
1384 class TestCreateWorkflow(unittest.TestCase):
1385     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1386     expect_workflow = StripYAMLComments(
1387         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1388
1389     def setUp(self):
1390         cwltool.process._names = set()
1391         arvados_cwl.arvdocker.arv_docker_clear_cache()
1392
1393     @stubs
1394     def test_create(self, stubs):
1395         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1396         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1397
1398         exited = arvados_cwl.main(
1399             ["--create-workflow", "--debug",
1400              "--api=containers",
1401              "--project-uuid", project_uuid,
1402              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1403             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1404
1405         stubs.api.pipeline_templates().create.refute_called()
1406         stubs.api.container_requests().create.refute_called()
1407
1408         body = {
1409             "workflow": {
1410                 "owner_uuid": project_uuid,
1411                 "name": "submit_wf.cwl",
1412                 "description": "",
1413                 "definition": self.expect_workflow,
1414             }
1415         }
1416         stubs.api.workflows().create.assert_called_with(
1417             body=JsonDiffMatcher(body))
1418
1419         self.assertEqual(stubs.capture_stdout.getvalue(),
1420                          stubs.expect_workflow_uuid + '\n')
1421         self.assertEqual(exited, 0)
1422
1423     @stubs
1424     def test_create_name(self, stubs):
1425         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1426         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1427
1428         exited = arvados_cwl.main(
1429             ["--create-workflow", "--debug",
1430              "--api=containers",
1431              "--project-uuid", project_uuid,
1432              "--name", "testing 123",
1433              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1434             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1435
1436         stubs.api.pipeline_templates().create.refute_called()
1437         stubs.api.container_requests().create.refute_called()
1438
1439         body = {
1440             "workflow": {
1441                 "owner_uuid": project_uuid,
1442                 "name": "testing 123",
1443                 "description": "",
1444                 "definition": self.expect_workflow,
1445             }
1446         }
1447         stubs.api.workflows().create.assert_called_with(
1448             body=JsonDiffMatcher(body))
1449
1450         self.assertEqual(stubs.capture_stdout.getvalue(),
1451                          stubs.expect_workflow_uuid + '\n')
1452         self.assertEqual(exited, 0)
1453
1454
1455     @stubs
1456     def test_update(self, stubs):
1457         exited = arvados_cwl.main(
1458             ["--update-workflow", self.existing_workflow_uuid,
1459              "--debug",
1460              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1461             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1462
1463         body = {
1464             "workflow": {
1465                 "name": "submit_wf.cwl",
1466                 "description": "",
1467                 "definition": self.expect_workflow,
1468             }
1469         }
1470         stubs.api.workflows().update.assert_called_with(
1471             uuid=self.existing_workflow_uuid,
1472             body=JsonDiffMatcher(body))
1473         self.assertEqual(stubs.capture_stdout.getvalue(),
1474                          self.existing_workflow_uuid + '\n')
1475         self.assertEqual(exited, 0)
1476
1477     @stubs
1478     def test_update_name(self, stubs):
1479         exited = arvados_cwl.main(
1480             ["--update-workflow", self.existing_workflow_uuid,
1481              "--debug", "--name", "testing 123",
1482              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1483             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1484
1485         body = {
1486             "workflow": {
1487                 "name": "testing 123",
1488                 "description": "",
1489                 "definition": self.expect_workflow,
1490             }
1491         }
1492         stubs.api.workflows().update.assert_called_with(
1493             uuid=self.existing_workflow_uuid,
1494             body=JsonDiffMatcher(body))
1495         self.assertEqual(stubs.capture_stdout.getvalue(),
1496                          self.existing_workflow_uuid + '\n')
1497         self.assertEqual(exited, 0)
1498
1499     @stubs
1500     def test_create_collection_per_tool(self, stubs):
1501         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1502         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1503
1504         exited = arvados_cwl.main(
1505             ["--create-workflow", "--debug",
1506              "--api=containers",
1507              "--project-uuid", project_uuid,
1508              "tests/collection_per_tool/collection_per_tool.cwl"],
1509             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1510
1511         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1512         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1513
1514         body = {
1515             "workflow": {
1516                 "owner_uuid": project_uuid,
1517                 "name": "collection_per_tool.cwl",
1518                 "description": "",
1519                 "definition": expect_workflow,
1520             }
1521         }
1522         stubs.api.workflows().create.assert_called_with(
1523             body=JsonDiffMatcher(body))
1524
1525         self.assertEqual(stubs.capture_stdout.getvalue(),
1526                          stubs.expect_workflow_uuid + '\n')
1527         self.assertEqual(exited, 0)
1528
1529     @stubs
1530     def test_create_with_imports(self, stubs):
1531         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1532         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1533
1534         exited = arvados_cwl.main(
1535             ["--create-workflow", "--debug",
1536              "--api=containers",
1537              "--project-uuid", project_uuid,
1538              "tests/wf/feddemo/feddemo.cwl"],
1539             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1540
1541         stubs.api.pipeline_templates().create.refute_called()
1542         stubs.api.container_requests().create.refute_called()
1543
1544         self.assertEqual(stubs.capture_stdout.getvalue(),
1545                          stubs.expect_workflow_uuid + '\n')
1546         self.assertEqual(exited, 0)
1547
1548     @stubs
1549     def test_create_with_no_input(self, stubs):
1550         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1551         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1552
1553         exited = arvados_cwl.main(
1554             ["--create-workflow", "--debug",
1555              "--api=containers",
1556              "--project-uuid", project_uuid,
1557              "tests/wf/revsort/revsort.cwl"],
1558             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1559
1560         stubs.api.pipeline_templates().create.refute_called()
1561         stubs.api.container_requests().create.refute_called()
1562
1563         self.assertEqual(stubs.capture_stdout.getvalue(),
1564                          stubs.expect_workflow_uuid + '\n')
1565         self.assertEqual(exited, 0)