sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=0',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343         arvados_cwl.arvdocker.arv_docker_clear_cache()
 344
 345     @stubs
 346     def test_error_when_multiple_storage_classes_specified(self, stubs):
 347         storage_classes = "foo,bar"
 348         exited = arvados_cwl.main(
 349                 ["--debug", "--storage-classes", storage_classes,
 350                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351                 sys.stdin, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 1)
 353
 354     @mock.patch("time.sleep")
 355     @stubs
 356     def test_submit_invalid_runner_ram(self, stubs, tm):
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 361         self.assertEqual(exited, 1)
 362
 363
 364     @stubs
 365     def test_submit_container(self, stubs):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--api=containers", "--debug",
 368                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 370
 371         stubs.api.collections().create.assert_has_calls([
 372             mock.call(body=JsonDiffMatcher({
 373                 'manifest_text':
 374                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 375                 'replication_desired': None,
 376                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 377             }), ensure_unique_name=False),
 378             mock.call(body=JsonDiffMatcher({
 379                 'manifest_text':
 380                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 381                 'replication_desired': None,
 382                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 383             }), ensure_unique_name=False),
 384             ])
 385
 386         expect_container = copy.deepcopy(stubs.expect_container_spec)
 387         stubs.api.container_requests().create.assert_called_with(
 388             body=JsonDiffMatcher(expect_container))
 389         self.assertEqual(stubs.capture_stdout.getvalue(),
 390                          stubs.expect_container_request_uuid + '\n')
 391         self.assertEqual(exited, 0)
 392
 393
 394     @stubs
 395     def test_submit_container_tool(self, stubs):
 396         # test for issue #16139
 397         exited = arvados_cwl.main(
 398             ["--submit", "--no-wait", "--api=containers", "--debug",
 399                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 400             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 401
 402         self.assertEqual(stubs.capture_stdout.getvalue(),
 403                          stubs.expect_container_request_uuid + '\n')
 404         self.assertEqual(exited, 0)
 405
 406     @stubs
 407     def test_submit_container_no_reuse(self, stubs):
 408         exited = arvados_cwl.main(
 409             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 410                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 412
 413         expect_container = copy.deepcopy(stubs.expect_container_spec)
 414         expect_container["command"] = [
 415             'arvados-cwl-runner', '--local', '--api=containers',
 416             '--no-log-timestamps', '--disable-validate', '--disable-color',
 417             '--eval-timeout=20', '--thread-count=0',
 418             '--disable-reuse', "--collection-cache-size=256",
 419             '--debug', '--on-error=continue',
 420             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 421         expect_container["use_existing"] = False
 422
 423         stubs.api.container_requests().create.assert_called_with(
 424             body=JsonDiffMatcher(expect_container))
 425         self.assertEqual(stubs.capture_stdout.getvalue(),
 426                          stubs.expect_container_request_uuid + '\n')
 427         self.assertEqual(exited, 0)
 428
 429     @stubs
 430     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug",
 433              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435         self.assertEqual(exited, 0)
 436
 437         expect_container = copy.deepcopy(stubs.expect_container_spec)
 438         expect_container["command"] = [
 439             'arvados-cwl-runner', '--local', '--api=containers',
 440             '--no-log-timestamps', '--disable-validate', '--disable-color',
 441             '--eval-timeout=20', '--thread-count=0',
 442             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 443             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 444         expect_container["use_existing"] = False
 445         expect_container["name"] = "submit_wf_no_reuse.cwl"
 446         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 447             {
 448                 "class": "http://arvados.org/cwl#ReuseRequirement",
 449                 "enableReuse": False,
 450             },
 451         ]
 452         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 453             "arv": "http://arvados.org/cwl#",
 454             "cwltool": "http://commonwl.org/cwltool#"
 455         }
 456
 457         stubs.api.container_requests().create.assert_called_with(
 458             body=JsonDiffMatcher(expect_container))
 459         self.assertEqual(stubs.capture_stdout.getvalue(),
 460                          stubs.expect_container_request_uuid + '\n')
 461
 462
 463     @stubs
 464     def test_submit_container_on_error(self, stubs):
 465         exited = arvados_cwl.main(
 466             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 467                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 468             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 469
 470         expect_container = copy.deepcopy(stubs.expect_container_spec)
 471         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 472                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 473                                        '--eval-timeout=20', '--thread-count=0',
 474                                        '--enable-reuse', "--collection-cache-size=256",
 475                                        '--debug', '--on-error=stop',
 476                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 477
 478         stubs.api.container_requests().create.assert_called_with(
 479             body=JsonDiffMatcher(expect_container))
 480         self.assertEqual(stubs.capture_stdout.getvalue(),
 481                          stubs.expect_container_request_uuid + '\n')
 482         self.assertEqual(exited, 0)
 483
 484     @stubs
 485     def test_submit_container_output_name(self, stubs):
 486         output_name = "test_output_name"
 487
 488         exited = arvados_cwl.main(
 489             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 490                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 491             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 492
 493         expect_container = copy.deepcopy(stubs.expect_container_spec)
 494         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 495                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 496                                        '--eval-timeout=20', '--thread-count=0',
 497                                        '--enable-reuse', "--collection-cache-size=256",
 498                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 499                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 500         expect_container["output_name"] = output_name
 501
 502         stubs.api.container_requests().create.assert_called_with(
 503             body=JsonDiffMatcher(expect_container))
 504         self.assertEqual(stubs.capture_stdout.getvalue(),
 505                          stubs.expect_container_request_uuid + '\n')
 506         self.assertEqual(exited, 0)
 507
 508     @stubs
 509     def test_submit_storage_classes(self, stubs):
 510         exited = arvados_cwl.main(
 511             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 512                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514
 515         expect_container = copy.deepcopy(stubs.expect_container_spec)
 516         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 517                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 518                                        '--eval-timeout=20', '--thread-count=0',
 519                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 520                                        "--storage-classes=foo", '--on-error=continue',
 521                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 522
 523         stubs.api.container_requests().create.assert_called_with(
 524             body=JsonDiffMatcher(expect_container))
 525         self.assertEqual(stubs.capture_stdout.getvalue(),
 526                          stubs.expect_container_request_uuid + '\n')
 527         self.assertEqual(exited, 0)
 528
 529     @mock.patch("cwltool.task_queue.TaskQueue")
 530     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 531     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 532     @stubs
 533     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 534         final_output_c = arvados.collection.Collection()
 535         make_output.return_value = ({},final_output_c)
 536
 537         def set_final_output(job_order, output_callback, runtimeContext):
 538             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 539             return []
 540         job.side_effect = set_final_output
 541
 542         exited = arvados_cwl.main(
 543             ["--debug", "--local", "--storage-classes=foo",
 544                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 545             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 546
 547         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 548         self.assertEqual(exited, 0)
 549
 550     @mock.patch("cwltool.task_queue.TaskQueue")
 551     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 552     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 553     @stubs
 554     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 555         final_output_c = arvados.collection.Collection()
 556         make_output.return_value = ({},final_output_c)
 557
 558         def set_final_output(job_order, output_callback, runtimeContext):
 559             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 560             return []
 561         job.side_effect = set_final_output
 562
 563         exited = arvados_cwl.main(
 564             ["--debug", "--local",
 565                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 566             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 567
 568         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 569         self.assertEqual(exited, 0)
 570
 571     @stubs
 572     def test_submit_container_output_ttl(self, stubs):
 573         exited = arvados_cwl.main(
 574             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 575                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 576             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 577
 578         expect_container = copy.deepcopy(stubs.expect_container_spec)
 579         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 580                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 581                                        '--eval-timeout=20', '--thread-count=0',
 582                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 583                                        '--on-error=continue',
 584                                        "--intermediate-output-ttl=3600",
 585                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586
 587         stubs.api.container_requests().create.assert_called_with(
 588             body=JsonDiffMatcher(expect_container))
 589         self.assertEqual(stubs.capture_stdout.getvalue(),
 590                          stubs.expect_container_request_uuid + '\n')
 591         self.assertEqual(exited, 0)
 592
 593     @stubs
 594     def test_submit_container_trash_intermediate(self, stubs):
 595         exited = arvados_cwl.main(
 596             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 597                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 598             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 599
 600
 601         expect_container = copy.deepcopy(stubs.expect_container_spec)
 602         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 603                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 604                                        '--eval-timeout=20', '--thread-count=0',
 605                                        '--enable-reuse', "--collection-cache-size=256",
 606                                        '--debug', '--on-error=continue',
 607                                        "--trash-intermediate",
 608                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 609
 610         stubs.api.container_requests().create.assert_called_with(
 611             body=JsonDiffMatcher(expect_container))
 612         self.assertEqual(stubs.capture_stdout.getvalue(),
 613                          stubs.expect_container_request_uuid + '\n')
 614         self.assertEqual(exited, 0)
 615
 616     @stubs
 617     def test_submit_container_output_tags(self, stubs):
 618         output_tags = "tag0,tag1,tag2"
 619
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 622                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 623             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 627                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 628                                        '--eval-timeout=20', '--thread-count=0',
 629                                        '--enable-reuse', "--collection-cache-size=256",
 630                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 631                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 632
 633         stubs.api.container_requests().create.assert_called_with(
 634             body=JsonDiffMatcher(expect_container))
 635         self.assertEqual(stubs.capture_stdout.getvalue(),
 636                          stubs.expect_container_request_uuid + '\n')
 637         self.assertEqual(exited, 0)
 638
 639     @stubs
 640     def test_submit_container_runner_ram(self, stubs):
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 643                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 645
 646         expect_container = copy.deepcopy(stubs.expect_container_spec)
 647         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(stubs.capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653         self.assertEqual(exited, 0)
 654
 655     @mock.patch("arvados.collection.CollectionReader")
 656     @mock.patch("time.sleep")
 657     @stubs
 658     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 659         collectionReader().exists.return_value = True
 660         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug",
 663              "tests/wf/submit_keepref_wf.cwl"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 665         self.assertEqual(exited, 0)
 666
 667     @mock.patch("arvados.collection.CollectionReader")
 668     @mock.patch("time.sleep")
 669     @stubs
 670     def test_submit_keepref(self, stubs, tm, reader):
 671         with open("tests/wf/expect_arvworkflow.cwl") as f:
 672             reader().open().__enter__().read.return_value = f.read()
 673
 674         exited = arvados_cwl.main(
 675             ["--submit", "--no-wait", "--api=containers", "--debug",
 676              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 677             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 678
 679         expect_container = {
 680             'priority': 500,
 681             'mounts': {
 682                 '/var/spool/cwl': {
 683                     'writable': True,
 684                     'kind': 'collection'
 685                 },
 686                 'stdout': {
 687                     'path': '/var/spool/cwl/cwl.output.json',
 688                     'kind': 'file'
 689                 },
 690                 '/var/lib/cwl/workflow': {
 691                     'portable_data_hash': '99999999999999999999999999999994+99',
 692                     'kind': 'collection'
 693                 },
 694                 '/var/lib/cwl/cwl.input.json': {
 695                     'content': {
 696                         'x': 'XxX'
 697                     },
 698                     'kind': 'json'
 699                 }
 700             }, 'state': 'Committed',
 701             'output_path': '/var/spool/cwl',
 702             'name': 'expect_arvworkflow.cwl#main',
 703             'container_image': '999999999999999999999999999999d3+99',
 704             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 705                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 706                         '--eval-timeout=20', '--thread-count=0',
 707                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 708                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 709             'cwd': '/var/spool/cwl',
 710             'runtime_constraints': {
 711                 'API': True,
 712                 'vcpus': 1,
 713                 'ram': 1342177280
 714             },
 715             'use_existing': False,
 716             'properties': {},
 717             'secret_mounts': {}
 718         }
 719
 720         stubs.api.container_requests().create.assert_called_with(
 721             body=JsonDiffMatcher(expect_container))
 722         self.assertEqual(stubs.capture_stdout.getvalue(),
 723                          stubs.expect_container_request_uuid + '\n')
 724         self.assertEqual(exited, 0)
 725
 726     @mock.patch("time.sleep")
 727     @stubs
 728     def test_submit_arvworkflow(self, stubs, tm):
 729         with open("tests/wf/expect_arvworkflow.cwl") as f:
 730             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 731
 732         exited = arvados_cwl.main(
 733             ["--submit", "--no-wait", "--api=containers", "--debug",
 734              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 735             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 736
 737         expect_container = {
 738             'priority': 500,
 739             'mounts': {
 740                 '/var/spool/cwl': {
 741                     'writable': True,
 742                     'kind': 'collection'
 743                 },
 744                 'stdout': {
 745                     'path': '/var/spool/cwl/cwl.output.json',
 746                     'kind': 'file'
 747                 },
 748                 '/var/lib/cwl/workflow.json': {
 749                     'kind': 'json',
 750                     'content': {
 751                         'cwlVersion': 'v1.0',
 752                         '$graph': [
 753                             {
 754                                 'id': '#main',
 755                                 'inputs': [
 756                                     {'type': 'string', 'id': '#main/x'}
 757                                 ],
 758                                 'steps': [
 759                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 760                                      'run': '#submit_tool.cwl',
 761                                      'id': '#main/step1',
 762                                      'out': []}
 763                                 ],
 764                                 'class': 'Workflow',
 765                                 'outputs': []
 766                             },
 767                             {
 768                                 'inputs': [
 769                                     {
 770                                         'inputBinding': {'position': 1},
 771                                         'type': 'string',
 772                                         'id': '#submit_tool.cwl/x'}
 773                                 ],
 774                                 'requirements': [
 775                                     {
 776                                         'dockerPull': 'debian:buster-slim',
 777                                         'class': 'DockerRequirement',
 778                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 779                                     }
 780                                 ],
 781                                 'id': '#submit_tool.cwl',
 782                                 'outputs': [],
 783                                 'baseCommand': 'cat',
 784                                 'class': 'CommandLineTool'
 785                             }
 786                         ]
 787                     }
 788                 },
 789                 '/var/lib/cwl/cwl.input.json': {
 790                     'content': {
 791                         'x': 'XxX'
 792                     },
 793                     'kind': 'json'
 794                 }
 795             }, 'state': 'Committed',
 796             'output_path': '/var/spool/cwl',
 797             'name': 'a test workflow',
 798             'container_image': "999999999999999999999999999999d3+99",
 799             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 800                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 801                         '--eval-timeout=20', '--thread-count=0',
 802                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 803                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 804             'cwd': '/var/spool/cwl',
 805             'runtime_constraints': {
 806                 'API': True,
 807                 'vcpus': 1,
 808                 'ram': 1342177280
 809             },
 810             'use_existing': False,
 811             'properties': {
 812                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 813             },
 814             'secret_mounts': {}
 815         }
 816
 817         stubs.api.container_requests().create.assert_called_with(
 818             body=JsonDiffMatcher(expect_container))
 819         self.assertEqual(stubs.capture_stdout.getvalue(),
 820                          stubs.expect_container_request_uuid + '\n')
 821         self.assertEqual(exited, 0)
 822
 823     @stubs
 824     def test_submit_container_name(self, stubs):
 825         exited = arvados_cwl.main(
 826             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 827                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 828             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 829
 830         expect_container = copy.deepcopy(stubs.expect_container_spec)
 831         expect_container["name"] = "hello container 123"
 832
 833         stubs.api.container_requests().create.assert_called_with(
 834             body=JsonDiffMatcher(expect_container))
 835         self.assertEqual(stubs.capture_stdout.getvalue(),
 836                          stubs.expect_container_request_uuid + '\n')
 837         self.assertEqual(exited, 0)
 838
 839     @stubs
 840     def test_submit_missing_input(self, stubs):
 841         exited = arvados_cwl.main(
 842             ["--submit", "--no-wait", "--api=containers", "--debug",
 843              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 844             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 845         self.assertEqual(exited, 0)
 846
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug",
 849              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851         self.assertEqual(exited, 1)
 852
 853     @stubs
 854     def test_submit_container_project(self, stubs):
 855         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 856         exited = arvados_cwl.main(
 857             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 858                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 859             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 860
 861         expect_container = copy.deepcopy(stubs.expect_container_spec)
 862         expect_container["owner_uuid"] = project_uuid
 863         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 864                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 865                                        "--eval-timeout=20", "--thread-count=0",
 866                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 867                                        '--on-error=continue',
 868                                        '--project-uuid='+project_uuid,
 869                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 870
 871         stubs.api.container_requests().create.assert_called_with(
 872             body=JsonDiffMatcher(expect_container))
 873         self.assertEqual(stubs.capture_stdout.getvalue(),
 874                          stubs.expect_container_request_uuid + '\n')
 875         self.assertEqual(exited, 0)
 876
 877     @stubs
 878     def test_submit_container_eval_timeout(self, stubs):
 879         exited = arvados_cwl.main(
 880             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 881                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 882             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 883
 884         expect_container = copy.deepcopy(stubs.expect_container_spec)
 885         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 886                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 887                                        '--eval-timeout=60.0', '--thread-count=0',
 888                                        '--enable-reuse', "--collection-cache-size=256",
 889                                        '--debug', '--on-error=continue',
 890                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 891
 892         stubs.api.container_requests().create.assert_called_with(
 893             body=JsonDiffMatcher(expect_container))
 894         self.assertEqual(stubs.capture_stdout.getvalue(),
 895                          stubs.expect_container_request_uuid + '\n')
 896         self.assertEqual(exited, 0)
 897
 898     @stubs
 899     def test_submit_container_collection_cache(self, stubs):
 900         exited = arvados_cwl.main(
 901             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 902                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904
 905         expect_container = copy.deepcopy(stubs.expect_container_spec)
 906         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 907                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 908                                        '--eval-timeout=20', '--thread-count=0',
 909                                        '--enable-reuse', "--collection-cache-size=500",
 910                                        '--debug', '--on-error=continue',
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 913
 914         stubs.api.container_requests().create.assert_called_with(
 915             body=JsonDiffMatcher(expect_container))
 916         self.assertEqual(stubs.capture_stdout.getvalue(),
 917                          stubs.expect_container_request_uuid + '\n')
 918         self.assertEqual(exited, 0)
 919
 920     @stubs
 921     def test_submit_container_thread_count(self, stubs):
 922         exited = arvados_cwl.main(
 923             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 924                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 925             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 926
 927         expect_container = copy.deepcopy(stubs.expect_container_spec)
 928         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 929                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 930                                        '--eval-timeout=20', '--thread-count=20',
 931                                        '--enable-reuse', "--collection-cache-size=256",
 932                                        '--debug', '--on-error=continue',
 933                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 934
 935         stubs.api.container_requests().create.assert_called_with(
 936             body=JsonDiffMatcher(expect_container))
 937         self.assertEqual(stubs.capture_stdout.getvalue(),
 938                          stubs.expect_container_request_uuid + '\n')
 939         self.assertEqual(exited, 0)
 940
 941     @stubs
 942     def test_submit_container_runner_image(self, stubs):
 943         exited = arvados_cwl.main(
 944             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 945                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 946             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 947
 948         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 949
 950         expect_container = copy.deepcopy(stubs.expect_container_spec)
 951         stubs.api.container_requests().create.assert_called_with(
 952             body=JsonDiffMatcher(expect_container))
 953         self.assertEqual(stubs.capture_stdout.getvalue(),
 954                          stubs.expect_container_request_uuid + '\n')
 955         self.assertEqual(exited, 0)
 956
 957     @stubs
 958     def test_submit_priority(self, stubs):
 959         exited = arvados_cwl.main(
 960             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 961                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 962             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 963
 964         stubs.expect_container_spec["priority"] = 669
 965
 966         expect_container = copy.deepcopy(stubs.expect_container_spec)
 967         stubs.api.container_requests().create.assert_called_with(
 968             body=JsonDiffMatcher(expect_container))
 969         self.assertEqual(stubs.capture_stdout.getvalue(),
 970                          stubs.expect_container_request_uuid + '\n')
 971         self.assertEqual(exited, 0)
 972
 973     @stubs
 974     def test_submit_wf_runner_resources(self, stubs):
 975         exited = arvados_cwl.main(
 976             ["--submit", "--no-wait", "--api=containers", "--debug",
 977                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 978             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 979
 980         expect_container = copy.deepcopy(stubs.expect_container_spec)
 981         expect_container["runtime_constraints"] = {
 982             "API": True,
 983             "vcpus": 2,
 984             "ram": (2000+512) * 2**20
 985         }
 986         expect_container["name"] = "submit_wf_runner_resources.cwl"
 987         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 988             {
 989                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 990                 "coresMin": 2,
 991                 "ramMin": 2000,
 992                 "keep_cache": 512
 993             }
 994         ]
 995         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 996             "arv": "http://arvados.org/cwl#",
 997         }
 998         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 999                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1000                         '--eval-timeout=20', '--thread-count=0',
1001                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1002                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1003
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     def tearDown(self):
1011         arvados_cwl.arvdocker.arv_docker_clear_cache()
1012
1013     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1014     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1015     @mock.patch("arvados.api")
1016     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1017         arvados_cwl.arvdocker.arv_docker_clear_cache()
1018
1019         arvrunner = mock.MagicMock()
1020         arvrunner.project_uuid = ""
1021         api.return_value = mock.MagicMock()
1022         arvrunner.api = api.return_value
1023         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1024                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1025                                                                         "link_class": "docker_image_repo+tag",
1026                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1027                                                                         "owner_uuid": "",
1028                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1029                                                             {"items": [{"created_at": "",
1030                                                                         "head_uuid": "",
1031                                                                         "link_class": "docker_image_hash",
1032                                                                         "name": "123456",
1033                                                                         "owner_uuid": "",
1034                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1035         )
1036         find_one_image_hash.return_value = "123456"
1037
1038         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1039                                                                               "owner_uuid": "",
1040                                                                               "manifest_text": "",
1041                                                                               "properties": ""
1042                                                                           }], "items_available": 1, "offset": 0},)
1043         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1044         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1045                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1046         self.assertEqual("9999999999999999999999999999999b+99",
1047                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1048
1049
1050     @stubs
1051     def test_submit_secrets(self, stubs):
1052         exited = arvados_cwl.main(
1053             ["--submit", "--no-wait", "--api=containers", "--debug",
1054                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1055             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1056
1057         expect_container = {
1058             "command": [
1059                 "arvados-cwl-runner",
1060                 "--local",
1061                 "--api=containers",
1062                 "--no-log-timestamps",
1063                 "--disable-validate",
1064                 "--disable-color",
1065                 "--eval-timeout=20",
1066                 '--thread-count=0',
1067                 "--enable-reuse",
1068                 "--collection-cache-size=256",
1069                 '--debug',
1070                 "--on-error=continue",
1071                 "/var/lib/cwl/workflow.json#main",
1072                 "/var/lib/cwl/cwl.input.json"
1073             ],
1074             "container_image": "999999999999999999999999999999d3+99",
1075             "cwd": "/var/spool/cwl",
1076             "mounts": {
1077                 "/var/lib/cwl/cwl.input.json": {
1078                     "content": {
1079                         "pw": {
1080                             "$include": "/secrets/s0"
1081                         }
1082                     },
1083                     "kind": "json"
1084                 },
1085                 "/var/lib/cwl/workflow.json": {
1086                     "content": {
1087                         "$graph": [
1088                             {
1089                                 "$namespaces": {
1090                                     "cwltool": "http://commonwl.org/cwltool#"
1091                                 },
1092                                 "arguments": [
1093                                     "md5sum",
1094                                     "example.conf"
1095                                 ],
1096                                 "class": "CommandLineTool",
1097                                 "hints": [
1098                                     {
1099                                         "class": "http://commonwl.org/cwltool#Secrets",
1100                                         "secrets": [
1101                                             "#secret_job.cwl/pw"
1102                                         ]
1103                                     }
1104                                 ],
1105                                 "id": "#secret_job.cwl",
1106                                 "inputs": [
1107                                     {
1108                                         "id": "#secret_job.cwl/pw",
1109                                         "type": "string"
1110                                     }
1111                                 ],
1112                                 "outputs": [
1113                                     {
1114                                         "id": "#secret_job.cwl/out",
1115                                         "type": "File",
1116                                         "outputBinding": {
1117                                               "glob": "hashed_example.txt"
1118                                         }
1119                                     }
1120                                 ],
1121                                 "stdout": "hashed_example.txt",
1122                                 "requirements": [
1123                                     {
1124                                         "class": "InitialWorkDirRequirement",
1125                                         "listing": [
1126                                             {
1127                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1128                                                 "entryname": "example.conf"
1129                                             }
1130                                         ]
1131                                     }
1132                                 ]
1133                             },
1134                             {
1135                                 "class": "Workflow",
1136                                 "hints": [
1137                                     {
1138                                         "class": "DockerRequirement",
1139                                         "dockerPull": "debian:buster-slim",
1140                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1141                                     },
1142                                     {
1143                                         "class": "http://commonwl.org/cwltool#Secrets",
1144                                         "secrets": [
1145                                             "#main/pw"
1146                                         ]
1147                                     }
1148                                 ],
1149                                 "id": "#main",
1150                                 "inputs": [
1151                                     {
1152                                         "id": "#main/pw",
1153                                         "type": "string"
1154                                     }
1155                                 ],
1156                                 "outputs": [
1157                                     {
1158                                         "id": "#main/out",
1159                                         "outputSource": "#main/step1/out",
1160                                         "type": "File"
1161                                     }
1162                                 ],
1163                                 "steps": [
1164                                     {
1165                                         "id": "#main/step1",
1166                                         "in": [
1167                                             {
1168                                                 "id": "#main/step1/pw",
1169                                                 "source": "#main/pw"
1170                                             }
1171                                         ],
1172                                         "out": [
1173                                             "#main/step1/out"
1174                                         ],
1175                                         "run": "#secret_job.cwl"
1176                                     }
1177                                 ]
1178                             }
1179                         ],
1180                         "cwlVersion": "v1.0"
1181                     },
1182                     "kind": "json"
1183                 },
1184                 "/var/spool/cwl": {
1185                     "kind": "collection",
1186                     "writable": True
1187                 },
1188                 "stdout": {
1189                     "kind": "file",
1190                     "path": "/var/spool/cwl/cwl.output.json"
1191                 }
1192             },
1193             "name": "secret_wf.cwl",
1194             "output_path": "/var/spool/cwl",
1195             "priority": 500,
1196             "properties": {},
1197             "runtime_constraints": {
1198                 "API": True,
1199                 "ram": 1342177280,
1200                 "vcpus": 1
1201             },
1202             "secret_mounts": {
1203                 "/secrets/s0": {
1204                     "content": "blorp",
1205                     "kind": "text"
1206                 }
1207             },
1208             "state": "Committed",
1209             "use_existing": False
1210         }
1211
1212         stubs.api.container_requests().create.assert_called_with(
1213             body=JsonDiffMatcher(expect_container))
1214         self.assertEqual(stubs.capture_stdout.getvalue(),
1215                          stubs.expect_container_request_uuid + '\n')
1216         self.assertEqual(exited, 0)
1217
1218     @stubs
1219     def test_submit_request_uuid(self, stubs):
1220         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1221         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1222
1223         stubs.api.container_requests().update().execute.return_value = {
1224             "uuid": stubs.expect_container_request_uuid,
1225             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1226             "state": "Queued"
1227         }
1228
1229         exited = arvados_cwl.main(
1230             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1231                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1232             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1233
1234         stubs.api.container_requests().update.assert_called_with(
1235             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1236         self.assertEqual(stubs.capture_stdout.getvalue(),
1237                          stubs.expect_container_request_uuid + '\n')
1238         self.assertEqual(exited, 0)
1239
1240     @stubs
1241     def test_submit_container_cluster_id(self, stubs):
1242         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1243
1244         exited = arvados_cwl.main(
1245             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1246                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1247             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1248
1249         expect_container = copy.deepcopy(stubs.expect_container_spec)
1250
1251         stubs.api.container_requests().create.assert_called_with(
1252             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1253         self.assertEqual(stubs.capture_stdout.getvalue(),
1254                          stubs.expect_container_request_uuid + '\n')
1255         self.assertEqual(exited, 0)
1256
1257     @stubs
1258     def test_submit_validate_cluster_id(self, stubs):
1259         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1260         exited = arvados_cwl.main(
1261             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1262              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1263             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1264         self.assertEqual(exited, 1)
1265
1266     @stubs
1267     def test_submit_validate_project_uuid(self, stubs):
1268         # Fails with bad cluster prefix
1269         exited = arvados_cwl.main(
1270             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1271              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1272             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1273         self.assertEqual(exited, 1)
1274
1275         # Project lookup fails
1276         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1277         exited = arvados_cwl.main(
1278             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1279              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1280             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1281         self.assertEqual(exited, 1)
1282
1283         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1284         exited = arvados_cwl.main(
1285             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1286              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1287             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1288         self.assertEqual(exited, 0)
1289
1290
1291     @mock.patch("arvados.collection.CollectionReader")
1292     @stubs
1293     def test_submit_uuid_inputs(self, stubs, collectionReader):
1294         collectionReader().exists.return_value = True
1295         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1296         def list_side_effect(**kwargs):
1297             m = mock.MagicMock()
1298             if "count" in kwargs:
1299                 m.execute.return_value = {"items": [
1300                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1301                 ]}
1302             else:
1303                 m.execute.return_value = {"items": []}
1304             return m
1305         stubs.api.collections().list.side_effect = list_side_effect
1306
1307         exited = arvados_cwl.main(
1308             ["--submit", "--no-wait", "--api=containers", "--debug",
1309                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1310             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1311
1312         expect_container = copy.deepcopy(stubs.expect_container_spec)
1313         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1314         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1315         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1316
1317         stubs.api.collections().list.assert_has_calls([
1318             mock.call(count='none',
1319                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1320                       select=['uuid', 'portable_data_hash'])])
1321         stubs.api.container_requests().create.assert_called_with(
1322             body=JsonDiffMatcher(expect_container))
1323         self.assertEqual(stubs.capture_stdout.getvalue(),
1324                          stubs.expect_container_request_uuid + '\n')
1325         self.assertEqual(exited, 0)
1326
1327     @stubs
1328     def test_submit_mismatched_uuid_inputs(self, stubs):
1329         def list_side_effect(**kwargs):
1330             m = mock.MagicMock()
1331             if "count" in kwargs:
1332                 m.execute.return_value = {"items": [
1333                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1334                 ]}
1335             else:
1336                 m.execute.return_value = {"items": []}
1337             return m
1338         stubs.api.collections().list.side_effect = list_side_effect
1339
1340         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1341             capture_stderr = StringIO()
1342             cwltool_logger = logging.getLogger('cwltool')
1343             stderr_logger = logging.StreamHandler(capture_stderr)
1344             cwltool_logger.addHandler(stderr_logger)
1345
1346             try:
1347                 exited = arvados_cwl.main(
1348                     ["--submit", "--no-wait", "--api=containers", "--debug",
1349                         "tests/wf/submit_wf.cwl", infile],
1350                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1351
1352                 self.assertEqual(exited, 1)
1353                 self.assertRegex(
1354                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1355                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1356             finally:
1357                 cwltool_logger.removeHandler(stderr_logger)
1358
1359     @mock.patch("arvados.collection.CollectionReader")
1360     @stubs
1361     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1362         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1363         capture_stderr = StringIO()
1364
1365         cwltool_logger = logging.getLogger('cwltool')
1366         stderr_logger = logging.StreamHandler(capture_stderr)
1367         cwltool_logger.addHandler(stderr_logger)
1368
1369         exited = arvados_cwl.main(
1370             ["--submit", "--no-wait", "--api=containers", "--debug",
1371                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1372             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1373
1374         try:
1375             self.assertEqual(exited, 1)
1376             self.assertRegex(
1377                 capture_stderr.getvalue(),
1378                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1379         finally:
1380             cwltool_logger.removeHandler(stderr_logger)
1381
1382
1383 class TestCreateWorkflow(unittest.TestCase):
1384     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1385     expect_workflow = StripYAMLComments(
1386         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1387
1388     def setUp(self):
1389         cwltool.process._names = set()
1390         arvados_cwl.arvdocker.arv_docker_clear_cache()
1391
1392     @stubs
1393     def test_create(self, stubs):
1394         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1395
1396         exited = arvados_cwl.main(
1397             ["--create-workflow", "--debug",
1398              "--api=containers",
1399              "--project-uuid", project_uuid,
1400              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1401             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1402
1403         stubs.api.pipeline_templates().create.refute_called()
1404         stubs.api.container_requests().create.refute_called()
1405
1406         body = {
1407             "workflow": {
1408                 "owner_uuid": project_uuid,
1409                 "name": "submit_wf.cwl",
1410                 "description": "",
1411                 "definition": self.expect_workflow,
1412             }
1413         }
1414         stubs.api.workflows().create.assert_called_with(
1415             body=JsonDiffMatcher(body))
1416
1417         self.assertEqual(stubs.capture_stdout.getvalue(),
1418                          stubs.expect_workflow_uuid + '\n')
1419         self.assertEqual(exited, 0)
1420
1421     @stubs
1422     def test_create_name(self, stubs):
1423         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1424
1425         exited = arvados_cwl.main(
1426             ["--create-workflow", "--debug",
1427              "--api=containers",
1428              "--project-uuid", project_uuid,
1429              "--name", "testing 123",
1430              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1431             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1432
1433         stubs.api.pipeline_templates().create.refute_called()
1434         stubs.api.container_requests().create.refute_called()
1435
1436         body = {
1437             "workflow": {
1438                 "owner_uuid": project_uuid,
1439                 "name": "testing 123",
1440                 "description": "",
1441                 "definition": self.expect_workflow,
1442             }
1443         }
1444         stubs.api.workflows().create.assert_called_with(
1445             body=JsonDiffMatcher(body))
1446
1447         self.assertEqual(stubs.capture_stdout.getvalue(),
1448                          stubs.expect_workflow_uuid + '\n')
1449         self.assertEqual(exited, 0)
1450
1451
1452     @stubs
1453     def test_update(self, stubs):
1454         exited = arvados_cwl.main(
1455             ["--update-workflow", self.existing_workflow_uuid,
1456              "--debug",
1457              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1458             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1459
1460         body = {
1461             "workflow": {
1462                 "name": "submit_wf.cwl",
1463                 "description": "",
1464                 "definition": self.expect_workflow,
1465             }
1466         }
1467         stubs.api.workflows().update.assert_called_with(
1468             uuid=self.existing_workflow_uuid,
1469             body=JsonDiffMatcher(body))
1470         self.assertEqual(stubs.capture_stdout.getvalue(),
1471                          self.existing_workflow_uuid + '\n')
1472         self.assertEqual(exited, 0)
1473
1474     @stubs
1475     def test_update_name(self, stubs):
1476         exited = arvados_cwl.main(
1477             ["--update-workflow", self.existing_workflow_uuid,
1478              "--debug", "--name", "testing 123",
1479              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1480             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1481
1482         body = {
1483             "workflow": {
1484                 "name": "testing 123",
1485                 "description": "",
1486                 "definition": self.expect_workflow,
1487             }
1488         }
1489         stubs.api.workflows().update.assert_called_with(
1490             uuid=self.existing_workflow_uuid,
1491             body=JsonDiffMatcher(body))
1492         self.assertEqual(stubs.capture_stdout.getvalue(),
1493                          self.existing_workflow_uuid + '\n')
1494         self.assertEqual(exited, 0)
1495
1496     @stubs
1497     def test_create_collection_per_tool(self, stubs):
1498         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1499
1500         exited = arvados_cwl.main(
1501             ["--create-workflow", "--debug",
1502              "--api=containers",
1503              "--project-uuid", project_uuid,
1504              "tests/collection_per_tool/collection_per_tool.cwl"],
1505             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1506
1507         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1508         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1509
1510         body = {
1511             "workflow": {
1512                 "owner_uuid": project_uuid,
1513                 "name": "collection_per_tool.cwl",
1514                 "description": "",
1515                 "definition": expect_workflow,
1516             }
1517         }
1518         stubs.api.workflows().create.assert_called_with(
1519             body=JsonDiffMatcher(body))
1520
1521         self.assertEqual(stubs.capture_stdout.getvalue(),
1522                          stubs.expect_workflow_uuid + '\n')
1523         self.assertEqual(exited, 0)
1524
1525     @stubs
1526     def test_create_with_imports(self, stubs):
1527         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1528
1529         exited = arvados_cwl.main(
1530             ["--create-workflow", "--debug",
1531              "--api=containers",
1532              "--project-uuid", project_uuid,
1533              "tests/wf/feddemo/feddemo.cwl"],
1534             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1535
1536         stubs.api.pipeline_templates().create.refute_called()
1537         stubs.api.container_requests().create.refute_called()
1538
1539         self.assertEqual(stubs.capture_stdout.getvalue(),
1540                          stubs.expect_workflow_uuid + '\n')
1541         self.assertEqual(exited, 0)
1542
1543     @stubs
1544     def test_create_with_no_input(self, stubs):
1545         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1546
1547         exited = arvados_cwl.main(
1548             ["--create-workflow", "--debug",
1549              "--api=containers",
1550              "--project-uuid", project_uuid,
1551              "tests/wf/revsort/revsort.cwl"],
1552             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1553
1554         stubs.api.pipeline_templates().create.refute_called()
1555         stubs.api.container_requests().create.refute_called()
1556
1557         self.assertEqual(stubs.capture_stdout.getvalue(),
1558                          stubs.expect_workflow_uuid + '\n')
1559         self.assertEqual(exited, 0)