sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98         stubs.api.config()["StorageClasses"].items.return_value = {
  99             "default": {
 100                 "Default": True
 101             }
 102         }.items()
 103
 104         class CollectionExecute(object):
 105             def __init__(self, exe):
 106                 self.exe = exe
 107             def execute(self, num_retries=None):
 108                 return self.exe
 109
 110         def collection_createstub(created_collections, body, ensure_unique_name=None):
 111             mt = body["manifest_text"].encode('utf-8')
 112             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 113             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 114             created_collections[uuid] = {
 115                 "uuid": uuid,
 116                 "portable_data_hash": pdh,
 117                 "manifest_text": mt.decode('utf-8')
 118             }
 119             return CollectionExecute(created_collections[uuid])
 120
 121         def collection_getstub(created_collections, uuid):
 122             for v in viewvalues(created_collections):
 123                 if uuid in (v["uuid"], v["portable_data_hash"]):
 124                     return CollectionExecute(v)
 125
 126         created_collections = {
 127             "99999999999999999999999999999998+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999998+99",
 130                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999997+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999997+99",
 135                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 136             },
 137             "99999999999999999999999999999994+99": {
 138                 "uuid": "",
 139                 "portable_data_hash": "99999999999999999999999999999994+99",
 140                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 144                 "portable_data_hash": "999999999999999999999999999999d3+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 149                 "portable_data_hash": "999999999999999999999999999999d4+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 154                 "portable_data_hash": "999999999999999999999999999999d5+99",
 155                 "manifest_text": ""
 156             },
 157             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 158                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 159                 "portable_data_hash": "999999999999999999999999999999d6+99",
 160                 "manifest_text": ""
 161             }
 162         }
 163         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 164         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 165
 166         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 167         stubs.api.jobs().create().execute.return_value = {
 168             "uuid": stubs.expect_job_uuid,
 169             "state": "Queued",
 170         }
 171
 172         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 173         stubs.api.container_requests().create().execute.return_value = {
 174             "uuid": stubs.expect_container_request_uuid,
 175             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 176             "state": "Queued"
 177         }
 178
 179         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 180         stubs.api.pipeline_templates().create().execute.return_value = {
 181             "uuid": stubs.expect_pipeline_template_uuid,
 182         }
 183         stubs.expect_job_spec = {
 184             'runtime_constraints': {
 185                 'docker_image': '999999999999999999999999999999d3+99',
 186                 'min_ram_mb_per_node': 1024
 187             },
 188             'script_parameters': {
 189                 'x': {
 190                     'basename': 'blorp.txt',
 191                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 192                     'class': 'File'
 193                 },
 194                 'y': {
 195                     'basename': '99999999999999999999999999999998+99',
 196                     'location': 'keep:99999999999999999999999999999998+99',
 197                     'class': 'Directory'
 198                 },
 199                 'z': {
 200                     'basename': 'anonymous',
 201                     "listing": [{
 202                         "basename": "renamed.txt",
 203                         "class": "File",
 204                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 205                         "size": 0
 206                     }],
 207                     'class': 'Directory'
 208                 },
 209                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 210             },
 211             'repository': 'arvados',
 212             'script_version': 'master',
 213             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 214             'script': 'cwl-runner'
 215         }
 216         stubs.pipeline_component = stubs.expect_job_spec.copy()
 217         stubs.expect_pipeline_instance = {
 218             'name': 'submit_wf.cwl',
 219             'state': 'RunningOnServer',
 220             'owner_uuid': None,
 221             "components": {
 222                 "cwl-runner": {
 223                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 224                     'script_parameters': {
 225                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 226                         'x': {"value": {
 227                             'basename': 'blorp.txt',
 228                             'class': 'File',
 229                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 230                             "size": 16
 231                         }},
 232                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 233                               'listing': [
 234                                   {
 235                                       'basename': 'renamed.txt',
 236                                       'class': 'File', 'location':
 237                                       'keep:99999999999999999999999999999998+99/file1.txt',
 238                                       'size': 0
 239                                   }
 240                               ]}},
 241                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 242                         'arv:debug': True,
 243                         'arv:enable_reuse': True,
 244                         'arv:on_error': 'continue'
 245                     },
 246                     'repository': 'arvados',
 247                     'script_version': 'master',
 248                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 249                     'script': 'cwl-runner',
 250                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 251                 }
 252             }
 253         }
 254         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 255         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 256         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 257         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 258         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 259             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 260             "state": "Queued"
 261         }
 262         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 263         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 264
 265         with open("tests/wf/submit_wf_packed.cwl") as f:
 266             expect_packed_workflow = yaml.round_trip_load(f)
 267
 268         stubs.expect_container_spec = {
 269             'priority': 500,
 270             'mounts': {
 271                 '/var/spool/cwl': {
 272                     'writable': True,
 273                     'kind': 'collection'
 274                 },
 275                 '/var/lib/cwl/workflow.json': {
 276                     'content': expect_packed_workflow,
 277                     'kind': 'json'
 278                 },
 279                 'stdout': {
 280                     'path': '/var/spool/cwl/cwl.output.json',
 281                     'kind': 'file'
 282                 },
 283                 '/var/lib/cwl/cwl.input.json': {
 284                     'kind': 'json',
 285                     'content': {
 286                         'y': {
 287                             'basename': '99999999999999999999999999999998+99',
 288                             'location': 'keep:99999999999999999999999999999998+99',
 289                             'class': 'Directory'},
 290                         'x': {
 291                             'basename': u'blorp.txt',
 292                             'class': 'File',
 293                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 294                             "size": 16
 295                         },
 296                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 297                             {'basename': 'renamed.txt',
 298                              'class': 'File',
 299                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 300                              'size': 0
 301                             }
 302                         ]}
 303                     },
 304                     'kind': 'json'
 305                 }
 306             },
 307             'secret_mounts': {},
 308             'state': 'Committed',
 309             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 310                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 311                         '--eval-timeout=20', '--thread-count=0',
 312                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 313                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 314             'name': 'submit_wf.cwl',
 315             'container_image': '999999999999999999999999999999d3+99',
 316             'output_path': '/var/spool/cwl',
 317             'cwd': '/var/spool/cwl',
 318             'runtime_constraints': {
 319                 'API': True,
 320                 'vcpus': 1,
 321                 'ram': (1024+256)*1024*1024
 322             },
 323             'use_existing': False,
 324             'properties': {},
 325             'secret_mounts': {}
 326         }
 327
 328         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 329         stubs.api.workflows().create().execute.return_value = {
 330             "uuid": stubs.expect_workflow_uuid,
 331         }
 332         def update_mock(**kwargs):
 333             stubs.updated_uuid = kwargs.get('uuid')
 334             return mock.DEFAULT
 335         stubs.api.workflows().update.side_effect = update_mock
 336         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 337             "uuid": stubs.updated_uuid,
 338         }
 339
 340         return func(self, stubs, *args, **kwargs)
 341     return wrapped
 342
 343
 344 class TestSubmit(unittest.TestCase):
 345
 346     def setUp(self):
 347         cwltool.process._names = set()
 348         arvados_cwl.arvdocker.arv_docker_clear_cache()
 349
 350
 351     @mock.patch("time.sleep")
 352     @stubs
 353     def test_submit_invalid_runner_ram(self, stubs, tm):
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 358         self.assertEqual(exited, 1)
 359
 360
 361     @stubs
 362     def test_submit_container(self, stubs):
 363         exited = arvados_cwl.main(
 364             ["--submit", "--no-wait", "--api=containers", "--debug",
 365                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 366             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 367
 368         stubs.api.collections().create.assert_has_calls([
 369             mock.call(body=JsonDiffMatcher({
 370                 'manifest_text':
 371                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 372                 'replication_desired': None,
 373                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 374             }), ensure_unique_name=False),
 375             mock.call(body=JsonDiffMatcher({
 376                 'manifest_text':
 377                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 378                 'replication_desired': None,
 379                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 380             }), ensure_unique_name=False),
 381             ])
 382
 383         expect_container = copy.deepcopy(stubs.expect_container_spec)
 384         stubs.api.container_requests().create.assert_called_with(
 385             body=JsonDiffMatcher(expect_container))
 386         self.assertEqual(stubs.capture_stdout.getvalue(),
 387                          stubs.expect_container_request_uuid + '\n')
 388         self.assertEqual(exited, 0)
 389
 390
 391     @stubs
 392     def test_submit_container_tool(self, stubs):
 393         # test for issue #16139
 394         exited = arvados_cwl.main(
 395             ["--submit", "--no-wait", "--api=containers", "--debug",
 396                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 397             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 398
 399         self.assertEqual(stubs.capture_stdout.getvalue(),
 400                          stubs.expect_container_request_uuid + '\n')
 401         self.assertEqual(exited, 0)
 402
 403     @stubs
 404     def test_submit_container_no_reuse(self, stubs):
 405         exited = arvados_cwl.main(
 406             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 407                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 408             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 409
 410         expect_container = copy.deepcopy(stubs.expect_container_spec)
 411         expect_container["command"] = [
 412             'arvados-cwl-runner', '--local', '--api=containers',
 413             '--no-log-timestamps', '--disable-validate', '--disable-color',
 414             '--eval-timeout=20', '--thread-count=0',
 415             '--disable-reuse', "--collection-cache-size=256",
 416             '--debug', '--on-error=continue',
 417             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 418         expect_container["use_existing"] = False
 419
 420         stubs.api.container_requests().create.assert_called_with(
 421             body=JsonDiffMatcher(expect_container))
 422         self.assertEqual(stubs.capture_stdout.getvalue(),
 423                          stubs.expect_container_request_uuid + '\n')
 424         self.assertEqual(exited, 0)
 425
 426     @stubs
 427     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 428         exited = arvados_cwl.main(
 429             ["--submit", "--no-wait", "--api=containers", "--debug",
 430              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 431             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 432         self.assertEqual(exited, 0)
 433
 434         expect_container = copy.deepcopy(stubs.expect_container_spec)
 435         expect_container["command"] = [
 436             'arvados-cwl-runner', '--local', '--api=containers',
 437             '--no-log-timestamps', '--disable-validate', '--disable-color',
 438             '--eval-timeout=20', '--thread-count=0',
 439             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 440             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 441         expect_container["use_existing"] = False
 442         expect_container["name"] = "submit_wf_no_reuse.cwl"
 443         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 444             {
 445                 "class": "http://arvados.org/cwl#ReuseRequirement",
 446                 "enableReuse": False,
 447             },
 448         ]
 449         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 450             "arv": "http://arvados.org/cwl#",
 451             "cwltool": "http://commonwl.org/cwltool#"
 452         }
 453
 454         stubs.api.container_requests().create.assert_called_with(
 455             body=JsonDiffMatcher(expect_container))
 456         self.assertEqual(stubs.capture_stdout.getvalue(),
 457                          stubs.expect_container_request_uuid + '\n')
 458
 459
 460     @stubs
 461     def test_submit_container_on_error(self, stubs):
 462         exited = arvados_cwl.main(
 463             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 464                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 465             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 466
 467         expect_container = copy.deepcopy(stubs.expect_container_spec)
 468         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 469                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 470                                        '--eval-timeout=20', '--thread-count=0',
 471                                        '--enable-reuse', "--collection-cache-size=256",
 472                                        '--debug', '--on-error=stop',
 473                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 474
 475         stubs.api.container_requests().create.assert_called_with(
 476             body=JsonDiffMatcher(expect_container))
 477         self.assertEqual(stubs.capture_stdout.getvalue(),
 478                          stubs.expect_container_request_uuid + '\n')
 479         self.assertEqual(exited, 0)
 480
 481     @stubs
 482     def test_submit_container_output_name(self, stubs):
 483         output_name = "test_output_name"
 484
 485         exited = arvados_cwl.main(
 486             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 487                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 488             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 489
 490         expect_container = copy.deepcopy(stubs.expect_container_spec)
 491         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 492                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 493                                        '--eval-timeout=20', '--thread-count=0',
 494                                        '--enable-reuse', "--collection-cache-size=256",
 495                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 496                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 497         expect_container["output_name"] = output_name
 498
 499         stubs.api.container_requests().create.assert_called_with(
 500             body=JsonDiffMatcher(expect_container))
 501         self.assertEqual(stubs.capture_stdout.getvalue(),
 502                          stubs.expect_container_request_uuid + '\n')
 503         self.assertEqual(exited, 0)
 504
 505     @stubs
 506     def test_submit_storage_classes(self, stubs):
 507         exited = arvados_cwl.main(
 508             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 509                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 510             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 511
 512         expect_container = copy.deepcopy(stubs.expect_container_spec)
 513         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 514                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 515                                        '--eval-timeout=20', '--thread-count=0',
 516                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 517                                        "--storage-classes=foo", '--on-error=continue',
 518                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 519
 520         stubs.api.container_requests().create.assert_called_with(
 521             body=JsonDiffMatcher(expect_container))
 522         self.assertEqual(stubs.capture_stdout.getvalue(),
 523                          stubs.expect_container_request_uuid + '\n')
 524         self.assertEqual(exited, 0)
 525
 526     @stubs
 527     def test_submit_multiple_storage_classes(self, stubs):
 528         exited = arvados_cwl.main(
 529             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 530                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 531             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 532
 533         expect_container = copy.deepcopy(stubs.expect_container_spec)
 534         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 535                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 536                                        '--eval-timeout=20', '--thread-count=0',
 537                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 538                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 539                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 540
 541         stubs.api.container_requests().create.assert_called_with(
 542             body=JsonDiffMatcher(expect_container))
 543         self.assertEqual(stubs.capture_stdout.getvalue(),
 544                          stubs.expect_container_request_uuid + '\n')
 545         self.assertEqual(exited, 0)
 546
 547     @mock.patch("cwltool.task_queue.TaskQueue")
 548     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 549     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 550     @stubs
 551     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 552         final_output_c = arvados.collection.Collection()
 553         make_output.return_value = ({},final_output_c)
 554
 555         def set_final_output(job_order, output_callback, runtimeContext):
 556             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 557             return []
 558         job.side_effect = set_final_output
 559
 560         exited = arvados_cwl.main(
 561             ["--debug", "--local", "--storage-classes=foo",
 562                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 563             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 564
 565         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 566         self.assertEqual(exited, 0)
 567
 568     @mock.patch("cwltool.task_queue.TaskQueue")
 569     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 570     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 571     @stubs
 572     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 573         final_output_c = arvados.collection.Collection()
 574         make_output.return_value = ({},final_output_c)
 575
 576         def set_final_output(job_order, output_callback, runtimeContext):
 577             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 578             return []
 579         job.side_effect = set_final_output
 580
 581         exited = arvados_cwl.main(
 582             ["--debug", "--local",
 583                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 584             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 585
 586         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 587         self.assertEqual(exited, 0)
 588
 589     @mock.patch("cwltool.task_queue.TaskQueue")
 590     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 591     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 592     @stubs
 593     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 594         final_output_c = arvados.collection.Collection()
 595         make_output.return_value = ({},final_output_c)
 596
 597         def set_final_output(job_order, output_callback, runtimeContext):
 598             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 599             return []
 600         job.side_effect = set_final_output
 601
 602         exited = arvados_cwl.main(
 603             ["--debug", "--local",
 604                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 605             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 606
 607         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 608         self.assertEqual(exited, 0)
 609
 610     @stubs
 611     def test_submit_container_output_ttl(self, stubs):
 612         exited = arvados_cwl.main(
 613             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 614                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 615             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 616
 617         expect_container = copy.deepcopy(stubs.expect_container_spec)
 618         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 619                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 620                                        '--eval-timeout=20', '--thread-count=0',
 621                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 622                                        '--on-error=continue',
 623                                        "--intermediate-output-ttl=3600",
 624                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 625
 626         stubs.api.container_requests().create.assert_called_with(
 627             body=JsonDiffMatcher(expect_container))
 628         self.assertEqual(stubs.capture_stdout.getvalue(),
 629                          stubs.expect_container_request_uuid + '\n')
 630         self.assertEqual(exited, 0)
 631
 632     @stubs
 633     def test_submit_container_trash_intermediate(self, stubs):
 634         exited = arvados_cwl.main(
 635             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 636                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 637             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 638
 639
 640         expect_container = copy.deepcopy(stubs.expect_container_spec)
 641         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 642                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 643                                        '--eval-timeout=20', '--thread-count=0',
 644                                        '--enable-reuse', "--collection-cache-size=256",
 645                                        '--debug', '--on-error=continue',
 646                                        "--trash-intermediate",
 647                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(stubs.capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653         self.assertEqual(exited, 0)
 654
 655     @stubs
 656     def test_submit_container_output_tags(self, stubs):
 657         output_tags = "tag0,tag1,tag2"
 658
 659         exited = arvados_cwl.main(
 660             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 661                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 662             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 663
 664         expect_container = copy.deepcopy(stubs.expect_container_spec)
 665         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 666                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 667                                        '--eval-timeout=20', '--thread-count=0',
 668                                        '--enable-reuse', "--collection-cache-size=256",
 669                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 670                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 671
 672         stubs.api.container_requests().create.assert_called_with(
 673             body=JsonDiffMatcher(expect_container))
 674         self.assertEqual(stubs.capture_stdout.getvalue(),
 675                          stubs.expect_container_request_uuid + '\n')
 676         self.assertEqual(exited, 0)
 677
 678     @stubs
 679     def test_submit_container_runner_ram(self, stubs):
 680         exited = arvados_cwl.main(
 681             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 682                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 683             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 684
 685         expect_container = copy.deepcopy(stubs.expect_container_spec)
 686         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 687
 688         stubs.api.container_requests().create.assert_called_with(
 689             body=JsonDiffMatcher(expect_container))
 690         self.assertEqual(stubs.capture_stdout.getvalue(),
 691                          stubs.expect_container_request_uuid + '\n')
 692         self.assertEqual(exited, 0)
 693
 694     @mock.patch("arvados.collection.CollectionReader")
 695     @mock.patch("time.sleep")
 696     @stubs
 697     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 698         collectionReader().exists.return_value = True
 699         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 700         exited = arvados_cwl.main(
 701             ["--submit", "--no-wait", "--api=containers", "--debug",
 702              "tests/wf/submit_keepref_wf.cwl"],
 703             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 704         self.assertEqual(exited, 0)
 705
 706     @mock.patch("arvados.collection.CollectionReader")
 707     @mock.patch("time.sleep")
 708     @stubs
 709     def test_submit_keepref(self, stubs, tm, reader):
 710         with open("tests/wf/expect_arvworkflow.cwl") as f:
 711             reader().open().__enter__().read.return_value = f.read()
 712
 713         exited = arvados_cwl.main(
 714             ["--submit", "--no-wait", "--api=containers", "--debug",
 715              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 716             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 717
 718         expect_container = {
 719             'priority': 500,
 720             'mounts': {
 721                 '/var/spool/cwl': {
 722                     'writable': True,
 723                     'kind': 'collection'
 724                 },
 725                 'stdout': {
 726                     'path': '/var/spool/cwl/cwl.output.json',
 727                     'kind': 'file'
 728                 },
 729                 '/var/lib/cwl/workflow': {
 730                     'portable_data_hash': '99999999999999999999999999999994+99',
 731                     'kind': 'collection'
 732                 },
 733                 '/var/lib/cwl/cwl.input.json': {
 734                     'content': {
 735                         'x': 'XxX'
 736                     },
 737                     'kind': 'json'
 738                 }
 739             }, 'state': 'Committed',
 740             'output_path': '/var/spool/cwl',
 741             'name': 'expect_arvworkflow.cwl#main',
 742             'container_image': '999999999999999999999999999999d3+99',
 743             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 744                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 745                         '--eval-timeout=20', '--thread-count=0',
 746                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 747                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 748             'cwd': '/var/spool/cwl',
 749             'runtime_constraints': {
 750                 'API': True,
 751                 'vcpus': 1,
 752                 'ram': 1342177280
 753             },
 754             'use_existing': False,
 755             'properties': {},
 756             'secret_mounts': {}
 757         }
 758
 759         stubs.api.container_requests().create.assert_called_with(
 760             body=JsonDiffMatcher(expect_container))
 761         self.assertEqual(stubs.capture_stdout.getvalue(),
 762                          stubs.expect_container_request_uuid + '\n')
 763         self.assertEqual(exited, 0)
 764
 765     @mock.patch("time.sleep")
 766     @stubs
 767     def test_submit_arvworkflow(self, stubs, tm):
 768         with open("tests/wf/expect_arvworkflow.cwl") as f:
 769             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 770
 771         exited = arvados_cwl.main(
 772             ["--submit", "--no-wait", "--api=containers", "--debug",
 773              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 774             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 775
 776         expect_container = {
 777             'priority': 500,
 778             'mounts': {
 779                 '/var/spool/cwl': {
 780                     'writable': True,
 781                     'kind': 'collection'
 782                 },
 783                 'stdout': {
 784                     'path': '/var/spool/cwl/cwl.output.json',
 785                     'kind': 'file'
 786                 },
 787                 '/var/lib/cwl/workflow.json': {
 788                     'kind': 'json',
 789                     'content': {
 790                         'cwlVersion': 'v1.0',
 791                         '$graph': [
 792                             {
 793                                 'id': '#main',
 794                                 'inputs': [
 795                                     {'type': 'string', 'id': '#main/x'}
 796                                 ],
 797                                 'steps': [
 798                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 799                                      'run': '#submit_tool.cwl',
 800                                      'id': '#main/step1',
 801                                      'out': []}
 802                                 ],
 803                                 'class': 'Workflow',
 804                                 'outputs': []
 805                             },
 806                             {
 807                                 'inputs': [
 808                                     {
 809                                         'inputBinding': {'position': 1},
 810                                         'type': 'string',
 811                                         'id': '#submit_tool.cwl/x'}
 812                                 ],
 813                                 'requirements': [
 814                                     {
 815                                         'dockerPull': 'debian:buster-slim',
 816                                         'class': 'DockerRequirement',
 817                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 818                                     }
 819                                 ],
 820                                 'id': '#submit_tool.cwl',
 821                                 'outputs': [],
 822                                 'baseCommand': 'cat',
 823                                 'class': 'CommandLineTool'
 824                             }
 825                         ]
 826                     }
 827                 },
 828                 '/var/lib/cwl/cwl.input.json': {
 829                     'content': {
 830                         'x': 'XxX'
 831                     },
 832                     'kind': 'json'
 833                 }
 834             }, 'state': 'Committed',
 835             'output_path': '/var/spool/cwl',
 836             'name': 'a test workflow',
 837             'container_image': "999999999999999999999999999999d3+99",
 838             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 839                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 840                         '--eval-timeout=20', '--thread-count=0',
 841                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 842                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 843             'cwd': '/var/spool/cwl',
 844             'runtime_constraints': {
 845                 'API': True,
 846                 'vcpus': 1,
 847                 'ram': 1342177280
 848             },
 849             'use_existing': False,
 850             'properties': {
 851                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 852             },
 853             'secret_mounts': {}
 854         }
 855
 856         stubs.api.container_requests().create.assert_called_with(
 857             body=JsonDiffMatcher(expect_container))
 858         self.assertEqual(stubs.capture_stdout.getvalue(),
 859                          stubs.expect_container_request_uuid + '\n')
 860         self.assertEqual(exited, 0)
 861
 862     @stubs
 863     def test_submit_container_name(self, stubs):
 864         exited = arvados_cwl.main(
 865             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 866                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 867             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 868
 869         expect_container = copy.deepcopy(stubs.expect_container_spec)
 870         expect_container["name"] = "hello container 123"
 871
 872         stubs.api.container_requests().create.assert_called_with(
 873             body=JsonDiffMatcher(expect_container))
 874         self.assertEqual(stubs.capture_stdout.getvalue(),
 875                          stubs.expect_container_request_uuid + '\n')
 876         self.assertEqual(exited, 0)
 877
 878     @stubs
 879     def test_submit_missing_input(self, stubs):
 880         exited = arvados_cwl.main(
 881             ["--submit", "--no-wait", "--api=containers", "--debug",
 882              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 883             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 884         self.assertEqual(exited, 0)
 885
 886         exited = arvados_cwl.main(
 887             ["--submit", "--no-wait", "--api=containers", "--debug",
 888              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 889             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 890         self.assertEqual(exited, 1)
 891
 892     @stubs
 893     def test_submit_container_project(self, stubs):
 894         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 895         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 896         exited = arvados_cwl.main(
 897             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 898                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 899             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 900
 901         expect_container = copy.deepcopy(stubs.expect_container_spec)
 902         expect_container["owner_uuid"] = project_uuid
 903         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 904                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 905                                        "--eval-timeout=20", "--thread-count=0",
 906                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 907                                        '--on-error=continue',
 908                                        '--project-uuid='+project_uuid,
 909                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 910
 911         stubs.api.container_requests().create.assert_called_with(
 912             body=JsonDiffMatcher(expect_container))
 913         self.assertEqual(stubs.capture_stdout.getvalue(),
 914                          stubs.expect_container_request_uuid + '\n')
 915         self.assertEqual(exited, 0)
 916
 917     @stubs
 918     def test_submit_container_eval_timeout(self, stubs):
 919         exited = arvados_cwl.main(
 920             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 921                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 922             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 923
 924         expect_container = copy.deepcopy(stubs.expect_container_spec)
 925         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 926                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 927                                        '--eval-timeout=60.0', '--thread-count=0',
 928                                        '--enable-reuse', "--collection-cache-size=256",
 929                                        '--debug', '--on-error=continue',
 930                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 931
 932         stubs.api.container_requests().create.assert_called_with(
 933             body=JsonDiffMatcher(expect_container))
 934         self.assertEqual(stubs.capture_stdout.getvalue(),
 935                          stubs.expect_container_request_uuid + '\n')
 936         self.assertEqual(exited, 0)
 937
 938     @stubs
 939     def test_submit_container_collection_cache(self, stubs):
 940         exited = arvados_cwl.main(
 941             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 942                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 943             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 944
 945         expect_container = copy.deepcopy(stubs.expect_container_spec)
 946         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 947                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 948                                        '--eval-timeout=20', '--thread-count=0',
 949                                        '--enable-reuse', "--collection-cache-size=500",
 950                                        '--debug', '--on-error=continue',
 951                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 952         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 953
 954         stubs.api.container_requests().create.assert_called_with(
 955             body=JsonDiffMatcher(expect_container))
 956         self.assertEqual(stubs.capture_stdout.getvalue(),
 957                          stubs.expect_container_request_uuid + '\n')
 958         self.assertEqual(exited, 0)
 959
 960     @stubs
 961     def test_submit_container_thread_count(self, stubs):
 962         exited = arvados_cwl.main(
 963             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 964                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 965             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 966
 967         expect_container = copy.deepcopy(stubs.expect_container_spec)
 968         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 969                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 970                                        '--eval-timeout=20', '--thread-count=20',
 971                                        '--enable-reuse', "--collection-cache-size=256",
 972                                        '--debug', '--on-error=continue',
 973                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 974
 975         stubs.api.container_requests().create.assert_called_with(
 976             body=JsonDiffMatcher(expect_container))
 977         self.assertEqual(stubs.capture_stdout.getvalue(),
 978                          stubs.expect_container_request_uuid + '\n')
 979         self.assertEqual(exited, 0)
 980
 981     @stubs
 982     def test_submit_container_runner_image(self, stubs):
 983         exited = arvados_cwl.main(
 984             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 985                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 986             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 987
 988         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 989
 990         expect_container = copy.deepcopy(stubs.expect_container_spec)
 991         stubs.api.container_requests().create.assert_called_with(
 992             body=JsonDiffMatcher(expect_container))
 993         self.assertEqual(stubs.capture_stdout.getvalue(),
 994                          stubs.expect_container_request_uuid + '\n')
 995         self.assertEqual(exited, 0)
 996
 997     @stubs
 998     def test_submit_priority(self, stubs):
 999         exited = arvados_cwl.main(
1000             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1001                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1002             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1003
1004         stubs.expect_container_spec["priority"] = 669
1005
1006         expect_container = copy.deepcopy(stubs.expect_container_spec)
1007         stubs.api.container_requests().create.assert_called_with(
1008             body=JsonDiffMatcher(expect_container))
1009         self.assertEqual(stubs.capture_stdout.getvalue(),
1010                          stubs.expect_container_request_uuid + '\n')
1011         self.assertEqual(exited, 0)
1012
1013     @stubs
1014     def test_submit_wf_runner_resources(self, stubs):
1015         exited = arvados_cwl.main(
1016             ["--submit", "--no-wait", "--api=containers", "--debug",
1017                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1018             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1019
1020         expect_container = copy.deepcopy(stubs.expect_container_spec)
1021         expect_container["runtime_constraints"] = {
1022             "API": True,
1023             "vcpus": 2,
1024             "ram": (2000+512) * 2**20
1025         }
1026         expect_container["name"] = "submit_wf_runner_resources.cwl"
1027         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1028             {
1029                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1030                 "coresMin": 2,
1031                 "ramMin": 2000,
1032                 "keep_cache": 512
1033             }
1034         ]
1035         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1036             "arv": "http://arvados.org/cwl#",
1037         }
1038         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1039                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1040                         '--eval-timeout=20', '--thread-count=0',
1041                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1042                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1043
1044         stubs.api.container_requests().create.assert_called_with(
1045             body=JsonDiffMatcher(expect_container))
1046         self.assertEqual(stubs.capture_stdout.getvalue(),
1047                          stubs.expect_container_request_uuid + '\n')
1048         self.assertEqual(exited, 0)
1049
1050     def tearDown(self):
1051         arvados_cwl.arvdocker.arv_docker_clear_cache()
1052
1053     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1054     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1055     @mock.patch("arvados.api")
1056     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1057         arvados_cwl.arvdocker.arv_docker_clear_cache()
1058
1059         arvrunner = mock.MagicMock()
1060         arvrunner.project_uuid = ""
1061         api.return_value = mock.MagicMock()
1062         arvrunner.api = api.return_value
1063         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1064                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1065                                                                         "link_class": "docker_image_repo+tag",
1066                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1067                                                                         "owner_uuid": "",
1068                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1069                                                             {"items": [{"created_at": "",
1070                                                                         "head_uuid": "",
1071                                                                         "link_class": "docker_image_hash",
1072                                                                         "name": "123456",
1073                                                                         "owner_uuid": "",
1074                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1075         )
1076         find_one_image_hash.return_value = "123456"
1077
1078         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1079                                                                               "owner_uuid": "",
1080                                                                               "manifest_text": "",
1081                                                                               "properties": ""
1082                                                                           }], "items_available": 1, "offset": 0},)
1083         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1084         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1085                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1086         self.assertEqual("9999999999999999999999999999999b+99",
1087                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1088
1089
1090     @stubs
1091     def test_submit_secrets(self, stubs):
1092         exited = arvados_cwl.main(
1093             ["--submit", "--no-wait", "--api=containers", "--debug",
1094                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1095             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1096
1097         expect_container = {
1098             "command": [
1099                 "arvados-cwl-runner",
1100                 "--local",
1101                 "--api=containers",
1102                 "--no-log-timestamps",
1103                 "--disable-validate",
1104                 "--disable-color",
1105                 "--eval-timeout=20",
1106                 '--thread-count=0',
1107                 "--enable-reuse",
1108                 "--collection-cache-size=256",
1109                 '--debug',
1110                 "--on-error=continue",
1111                 "/var/lib/cwl/workflow.json#main",
1112                 "/var/lib/cwl/cwl.input.json"
1113             ],
1114             "container_image": "999999999999999999999999999999d3+99",
1115             "cwd": "/var/spool/cwl",
1116             "mounts": {
1117                 "/var/lib/cwl/cwl.input.json": {
1118                     "content": {
1119                         "pw": {
1120                             "$include": "/secrets/s0"
1121                         }
1122                     },
1123                     "kind": "json"
1124                 },
1125                 "/var/lib/cwl/workflow.json": {
1126                     "content": {
1127                         "$graph": [
1128                             {
1129                                 "$namespaces": {
1130                                     "cwltool": "http://commonwl.org/cwltool#"
1131                                 },
1132                                 "arguments": [
1133                                     "md5sum",
1134                                     "example.conf"
1135                                 ],
1136                                 "class": "CommandLineTool",
1137                                 "hints": [
1138                                     {
1139                                         "class": "http://commonwl.org/cwltool#Secrets",
1140                                         "secrets": [
1141                                             "#secret_job.cwl/pw"
1142                                         ]
1143                                     }
1144                                 ],
1145                                 "id": "#secret_job.cwl",
1146                                 "inputs": [
1147                                     {
1148                                         "id": "#secret_job.cwl/pw",
1149                                         "type": "string"
1150                                     }
1151                                 ],
1152                                 "outputs": [
1153                                     {
1154                                         "id": "#secret_job.cwl/out",
1155                                         "type": "File",
1156                                         "outputBinding": {
1157                                               "glob": "hashed_example.txt"
1158                                         }
1159                                     }
1160                                 ],
1161                                 "stdout": "hashed_example.txt",
1162                                 "requirements": [
1163                                     {
1164                                         "class": "InitialWorkDirRequirement",
1165                                         "listing": [
1166                                             {
1167                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1168                                                 "entryname": "example.conf"
1169                                             }
1170                                         ]
1171                                     }
1172                                 ]
1173                             },
1174                             {
1175                                 "class": "Workflow",
1176                                 "hints": [
1177                                     {
1178                                         "class": "DockerRequirement",
1179                                         "dockerPull": "debian:buster-slim",
1180                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1181                                     },
1182                                     {
1183                                         "class": "http://commonwl.org/cwltool#Secrets",
1184                                         "secrets": [
1185                                             "#main/pw"
1186                                         ]
1187                                     }
1188                                 ],
1189                                 "id": "#main",
1190                                 "inputs": [
1191                                     {
1192                                         "id": "#main/pw",
1193                                         "type": "string"
1194                                     }
1195                                 ],
1196                                 "outputs": [
1197                                     {
1198                                         "id": "#main/out",
1199                                         "outputSource": "#main/step1/out",
1200                                         "type": "File"
1201                                     }
1202                                 ],
1203                                 "steps": [
1204                                     {
1205                                         "id": "#main/step1",
1206                                         "in": [
1207                                             {
1208                                                 "id": "#main/step1/pw",
1209                                                 "source": "#main/pw"
1210                                             }
1211                                         ],
1212                                         "out": [
1213                                             "#main/step1/out"
1214                                         ],
1215                                         "run": "#secret_job.cwl"
1216                                     }
1217                                 ]
1218                             }
1219                         ],
1220                         "cwlVersion": "v1.0"
1221                     },
1222                     "kind": "json"
1223                 },
1224                 "/var/spool/cwl": {
1225                     "kind": "collection",
1226                     "writable": True
1227                 },
1228                 "stdout": {
1229                     "kind": "file",
1230                     "path": "/var/spool/cwl/cwl.output.json"
1231                 }
1232             },
1233             "name": "secret_wf.cwl",
1234             "output_path": "/var/spool/cwl",
1235             "priority": 500,
1236             "properties": {},
1237             "runtime_constraints": {
1238                 "API": True,
1239                 "ram": 1342177280,
1240                 "vcpus": 1
1241             },
1242             "secret_mounts": {
1243                 "/secrets/s0": {
1244                     "content": "blorp",
1245                     "kind": "text"
1246                 }
1247             },
1248             "state": "Committed",
1249             "use_existing": False
1250         }
1251
1252         stubs.api.container_requests().create.assert_called_with(
1253             body=JsonDiffMatcher(expect_container))
1254         self.assertEqual(stubs.capture_stdout.getvalue(),
1255                          stubs.expect_container_request_uuid + '\n')
1256         self.assertEqual(exited, 0)
1257
1258     @stubs
1259     def test_submit_request_uuid(self, stubs):
1260         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1261         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1262
1263         stubs.api.container_requests().update().execute.return_value = {
1264             "uuid": stubs.expect_container_request_uuid,
1265             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1266             "state": "Queued"
1267         }
1268
1269         exited = arvados_cwl.main(
1270             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1271                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1272             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1273
1274         stubs.api.container_requests().update.assert_called_with(
1275             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1276         self.assertEqual(stubs.capture_stdout.getvalue(),
1277                          stubs.expect_container_request_uuid + '\n')
1278         self.assertEqual(exited, 0)
1279
1280     @stubs
1281     def test_submit_container_cluster_id(self, stubs):
1282         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1283
1284         exited = arvados_cwl.main(
1285             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1286                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1287             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1288
1289         expect_container = copy.deepcopy(stubs.expect_container_spec)
1290
1291         stubs.api.container_requests().create.assert_called_with(
1292             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1293         self.assertEqual(stubs.capture_stdout.getvalue(),
1294                          stubs.expect_container_request_uuid + '\n')
1295         self.assertEqual(exited, 0)
1296
1297     @stubs
1298     def test_submit_validate_cluster_id(self, stubs):
1299         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1300         exited = arvados_cwl.main(
1301             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1302              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1303             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1304         self.assertEqual(exited, 1)
1305
1306     @stubs
1307     def test_submit_validate_project_uuid(self, stubs):
1308         # Fails with bad cluster prefix
1309         exited = arvados_cwl.main(
1310             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1311              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1312             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1313         self.assertEqual(exited, 1)
1314
1315         # Project lookup fails
1316         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1317         exited = arvados_cwl.main(
1318             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1320             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1321         self.assertEqual(exited, 1)
1322
1323         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1324         exited = arvados_cwl.main(
1325             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1326              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1327             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1328         self.assertEqual(exited, 0)
1329
1330
1331     @mock.patch("arvados.collection.CollectionReader")
1332     @stubs
1333     def test_submit_uuid_inputs(self, stubs, collectionReader):
1334         collectionReader().exists.return_value = True
1335         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1336         def list_side_effect(**kwargs):
1337             m = mock.MagicMock()
1338             if "count" in kwargs:
1339                 m.execute.return_value = {"items": [
1340                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1341                 ]}
1342             else:
1343                 m.execute.return_value = {"items": []}
1344             return m
1345         stubs.api.collections().list.side_effect = list_side_effect
1346
1347         exited = arvados_cwl.main(
1348             ["--submit", "--no-wait", "--api=containers", "--debug",
1349                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1350             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1351
1352         expect_container = copy.deepcopy(stubs.expect_container_spec)
1353         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1354         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1355         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1356
1357         stubs.api.collections().list.assert_has_calls([
1358             mock.call(count='none',
1359                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1360                       select=['uuid', 'portable_data_hash'])])
1361         stubs.api.container_requests().create.assert_called_with(
1362             body=JsonDiffMatcher(expect_container))
1363         self.assertEqual(stubs.capture_stdout.getvalue(),
1364                          stubs.expect_container_request_uuid + '\n')
1365         self.assertEqual(exited, 0)
1366
1367     @stubs
1368     def test_submit_mismatched_uuid_inputs(self, stubs):
1369         def list_side_effect(**kwargs):
1370             m = mock.MagicMock()
1371             if "count" in kwargs:
1372                 m.execute.return_value = {"items": [
1373                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1374                 ]}
1375             else:
1376                 m.execute.return_value = {"items": []}
1377             return m
1378         stubs.api.collections().list.side_effect = list_side_effect
1379
1380         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1381             capture_stderr = StringIO()
1382             cwltool_logger = logging.getLogger('cwltool')
1383             stderr_logger = logging.StreamHandler(capture_stderr)
1384             cwltool_logger.addHandler(stderr_logger)
1385
1386             try:
1387                 exited = arvados_cwl.main(
1388                     ["--submit", "--no-wait", "--api=containers", "--debug",
1389                         "tests/wf/submit_wf.cwl", infile],
1390                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1391
1392                 self.assertEqual(exited, 1)
1393                 self.assertRegex(
1394                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1395                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1396             finally:
1397                 cwltool_logger.removeHandler(stderr_logger)
1398
1399     @mock.patch("arvados.collection.CollectionReader")
1400     @stubs
1401     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1402         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1403         capture_stderr = StringIO()
1404
1405         cwltool_logger = logging.getLogger('cwltool')
1406         stderr_logger = logging.StreamHandler(capture_stderr)
1407         cwltool_logger.addHandler(stderr_logger)
1408
1409         exited = arvados_cwl.main(
1410             ["--submit", "--no-wait", "--api=containers", "--debug",
1411                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1412             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1413
1414         try:
1415             self.assertEqual(exited, 1)
1416             self.assertRegex(
1417                 capture_stderr.getvalue(),
1418                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1419         finally:
1420             cwltool_logger.removeHandler(stderr_logger)
1421
1422
1423 class TestCreateWorkflow(unittest.TestCase):
1424     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1425     expect_workflow = StripYAMLComments(
1426         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1427
1428     def setUp(self):
1429         cwltool.process._names = set()
1430         arvados_cwl.arvdocker.arv_docker_clear_cache()
1431
1432     @stubs
1433     def test_create(self, stubs):
1434         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1435         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1436
1437         exited = arvados_cwl.main(
1438             ["--create-workflow", "--debug",
1439              "--api=containers",
1440              "--project-uuid", project_uuid,
1441              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1442             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1443
1444         stubs.api.pipeline_templates().create.refute_called()
1445         stubs.api.container_requests().create.refute_called()
1446
1447         body = {
1448             "workflow": {
1449                 "owner_uuid": project_uuid,
1450                 "name": "submit_wf.cwl",
1451                 "description": "",
1452                 "definition": self.expect_workflow,
1453             }
1454         }
1455         stubs.api.workflows().create.assert_called_with(
1456             body=JsonDiffMatcher(body))
1457
1458         self.assertEqual(stubs.capture_stdout.getvalue(),
1459                          stubs.expect_workflow_uuid + '\n')
1460         self.assertEqual(exited, 0)
1461
1462     @stubs
1463     def test_create_name(self, stubs):
1464         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1465         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1466
1467         exited = arvados_cwl.main(
1468             ["--create-workflow", "--debug",
1469              "--api=containers",
1470              "--project-uuid", project_uuid,
1471              "--name", "testing 123",
1472              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1473             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1474
1475         stubs.api.pipeline_templates().create.refute_called()
1476         stubs.api.container_requests().create.refute_called()
1477
1478         body = {
1479             "workflow": {
1480                 "owner_uuid": project_uuid,
1481                 "name": "testing 123",
1482                 "description": "",
1483                 "definition": self.expect_workflow,
1484             }
1485         }
1486         stubs.api.workflows().create.assert_called_with(
1487             body=JsonDiffMatcher(body))
1488
1489         self.assertEqual(stubs.capture_stdout.getvalue(),
1490                          stubs.expect_workflow_uuid + '\n')
1491         self.assertEqual(exited, 0)
1492
1493
1494     @stubs
1495     def test_update(self, stubs):
1496         exited = arvados_cwl.main(
1497             ["--update-workflow", self.existing_workflow_uuid,
1498              "--debug",
1499              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1500             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1501
1502         body = {
1503             "workflow": {
1504                 "name": "submit_wf.cwl",
1505                 "description": "",
1506                 "definition": self.expect_workflow,
1507             }
1508         }
1509         stubs.api.workflows().update.assert_called_with(
1510             uuid=self.existing_workflow_uuid,
1511             body=JsonDiffMatcher(body))
1512         self.assertEqual(stubs.capture_stdout.getvalue(),
1513                          self.existing_workflow_uuid + '\n')
1514         self.assertEqual(exited, 0)
1515
1516     @stubs
1517     def test_update_name(self, stubs):
1518         exited = arvados_cwl.main(
1519             ["--update-workflow", self.existing_workflow_uuid,
1520              "--debug", "--name", "testing 123",
1521              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1522             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1523
1524         body = {
1525             "workflow": {
1526                 "name": "testing 123",
1527                 "description": "",
1528                 "definition": self.expect_workflow,
1529             }
1530         }
1531         stubs.api.workflows().update.assert_called_with(
1532             uuid=self.existing_workflow_uuid,
1533             body=JsonDiffMatcher(body))
1534         self.assertEqual(stubs.capture_stdout.getvalue(),
1535                          self.existing_workflow_uuid + '\n')
1536         self.assertEqual(exited, 0)
1537
1538     @stubs
1539     def test_create_collection_per_tool(self, stubs):
1540         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1541         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1542
1543         exited = arvados_cwl.main(
1544             ["--create-workflow", "--debug",
1545              "--api=containers",
1546              "--project-uuid", project_uuid,
1547              "tests/collection_per_tool/collection_per_tool.cwl"],
1548             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1549
1550         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1551         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1552
1553         body = {
1554             "workflow": {
1555                 "owner_uuid": project_uuid,
1556                 "name": "collection_per_tool.cwl",
1557                 "description": "",
1558                 "definition": expect_workflow,
1559             }
1560         }
1561         stubs.api.workflows().create.assert_called_with(
1562             body=JsonDiffMatcher(body))
1563
1564         self.assertEqual(stubs.capture_stdout.getvalue(),
1565                          stubs.expect_workflow_uuid + '\n')
1566         self.assertEqual(exited, 0)
1567
1568     @stubs
1569     def test_create_with_imports(self, stubs):
1570         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1571         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1572
1573         exited = arvados_cwl.main(
1574             ["--create-workflow", "--debug",
1575              "--api=containers",
1576              "--project-uuid", project_uuid,
1577              "tests/wf/feddemo/feddemo.cwl"],
1578             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1579
1580         stubs.api.pipeline_templates().create.refute_called()
1581         stubs.api.container_requests().create.refute_called()
1582
1583         self.assertEqual(stubs.capture_stdout.getvalue(),
1584                          stubs.expect_workflow_uuid + '\n')
1585         self.assertEqual(exited, 0)
1586
1587     @stubs
1588     def test_create_with_no_input(self, stubs):
1589         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1590         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1591
1592         exited = arvados_cwl.main(
1593             ["--create-workflow", "--debug",
1594              "--api=containers",
1595              "--project-uuid", project_uuid,
1596              "tests/wf/revsort/revsort.cwl"],
1597             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1598
1599         stubs.api.pipeline_templates().create.refute_called()
1600         stubs.api.container_requests().create.refute_called()
1601
1602         self.assertEqual(stubs.capture_stdout.getvalue(),
1603                          stubs.expect_workflow_uuid + '\n')
1604         self.assertEqual(exited, 0)