sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90         stubs.api._rootDesc["revision"] = "20210628"
  91
  92         stubs.api.users().current().execute.return_value = {
  93             "uuid": stubs.fake_user_uuid,
  94         }
  95         stubs.api.collections().list().execute.return_value = {"items": []}
  96         stubs.api.containers().current().execute.return_value = {
  97             "uuid": stubs.fake_container_uuid,
  98         }
  99         stubs.api.config()["StorageClasses"].items.return_value = {
 100             "default": {
 101                 "Default": True
 102             }
 103         }.items()
 104
 105         class CollectionExecute(object):
 106             def __init__(self, exe):
 107                 self.exe = exe
 108             def execute(self, num_retries=None):
 109                 return self.exe
 110
 111         def collection_createstub(created_collections, body, ensure_unique_name=None):
 112             mt = body["manifest_text"].encode('utf-8')
 113             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 114             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 115             created_collections[uuid] = {
 116                 "uuid": uuid,
 117                 "portable_data_hash": pdh,
 118                 "manifest_text": mt.decode('utf-8')
 119             }
 120             return CollectionExecute(created_collections[uuid])
 121
 122         def collection_getstub(created_collections, uuid):
 123             for v in viewvalues(created_collections):
 124                 if uuid in (v["uuid"], v["portable_data_hash"]):
 125                     return CollectionExecute(v)
 126
 127         created_collections = {
 128             "99999999999999999999999999999998+99": {
 129                 "uuid": "",
 130                 "portable_data_hash": "99999999999999999999999999999998+99",
 131                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 132             },
 133             "99999999999999999999999999999997+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999997+99",
 136                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999994+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999994+99",
 141                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 142             },
 143             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 144                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 145                 "portable_data_hash": "999999999999999999999999999999d3+99",
 146                 "manifest_text": ""
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 150                 "portable_data_hash": "999999999999999999999999999999d4+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 155                 "portable_data_hash": "999999999999999999999999999999d5+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 160                 "portable_data_hash": "999999999999999999999999999999d6+99",
 161                 "manifest_text": ""
 162             }
 163         }
 164         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 165         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 166
 167         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 168         stubs.api.jobs().create().execute.return_value = {
 169             "uuid": stubs.expect_job_uuid,
 170             "state": "Queued",
 171         }
 172
 173         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 174         stubs.api.container_requests().create().execute.return_value = {
 175             "uuid": stubs.expect_container_request_uuid,
 176             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 177             "state": "Queued"
 178         }
 179
 180         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 181         stubs.api.pipeline_templates().create().execute.return_value = {
 182             "uuid": stubs.expect_pipeline_template_uuid,
 183         }
 184         stubs.expect_job_spec = {
 185             'runtime_constraints': {
 186                 'docker_image': '999999999999999999999999999999d3+99',
 187                 'min_ram_mb_per_node': 1024
 188             },
 189             'script_parameters': {
 190                 'x': {
 191                     'basename': 'blorp.txt',
 192                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 193                     'class': 'File'
 194                 },
 195                 'y': {
 196                     'basename': '99999999999999999999999999999998+99',
 197                     'location': 'keep:99999999999999999999999999999998+99',
 198                     'class': 'Directory'
 199                 },
 200                 'z': {
 201                     'basename': 'anonymous',
 202                     "listing": [{
 203                         "basename": "renamed.txt",
 204                         "class": "File",
 205                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 206                         "size": 0
 207                     }],
 208                     'class': 'Directory'
 209                 },
 210                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 211             },
 212             'repository': 'arvados',
 213             'script_version': 'master',
 214             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 215             'script': 'cwl-runner'
 216         }
 217         stubs.pipeline_component = stubs.expect_job_spec.copy()
 218         stubs.expect_pipeline_instance = {
 219             'name': 'submit_wf.cwl',
 220             'state': 'RunningOnServer',
 221             'owner_uuid': None,
 222             "components": {
 223                 "cwl-runner": {
 224                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 225                     'script_parameters': {
 226                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 227                         'x': {"value": {
 228                             'basename': 'blorp.txt',
 229                             'class': 'File',
 230                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 231                             "size": 16
 232                         }},
 233                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 234                               'listing': [
 235                                   {
 236                                       'basename': 'renamed.txt',
 237                                       'class': 'File', 'location':
 238                                       'keep:99999999999999999999999999999998+99/file1.txt',
 239                                       'size': 0
 240                                   }
 241                               ]}},
 242                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 243                         'arv:debug': True,
 244                         'arv:enable_reuse': True,
 245                         'arv:on_error': 'continue'
 246                     },
 247                     'repository': 'arvados',
 248                     'script_version': 'master',
 249                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 250                     'script': 'cwl-runner',
 251                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 252                 }
 253             }
 254         }
 255         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 256         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 257         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 258         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 259         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 260             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 261             "state": "Queued"
 262         }
 263         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 264         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 265
 266         with open("tests/wf/submit_wf_packed.cwl") as f:
 267             expect_packed_workflow = yaml.round_trip_load(f)
 268
 269         stubs.expect_container_spec = {
 270             'priority': 500,
 271             'mounts': {
 272                 '/var/spool/cwl': {
 273                     'writable': True,
 274                     'kind': 'collection'
 275                 },
 276                 '/var/lib/cwl/workflow.json': {
 277                     'content': expect_packed_workflow,
 278                     'kind': 'json'
 279                 },
 280                 'stdout': {
 281                     'path': '/var/spool/cwl/cwl.output.json',
 282                     'kind': 'file'
 283                 },
 284                 '/var/lib/cwl/cwl.input.json': {
 285                     'kind': 'json',
 286                     'content': {
 287                         'y': {
 288                             'basename': '99999999999999999999999999999998+99',
 289                             'location': 'keep:99999999999999999999999999999998+99',
 290                             'class': 'Directory'},
 291                         'x': {
 292                             'basename': u'blorp.txt',
 293                             'class': 'File',
 294                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 295                             "size": 16
 296                         },
 297                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 298                             {'basename': 'renamed.txt',
 299                              'class': 'File',
 300                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 301                              'size': 0
 302                             }
 303                         ]}
 304                     },
 305                     'kind': 'json'
 306                 }
 307             },
 308             'secret_mounts': {},
 309             'state': 'Committed',
 310             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 311                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 312                         '--eval-timeout=20', '--thread-count=0',
 313                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 314                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 315             'name': 'submit_wf.cwl',
 316             'container_image': '999999999999999999999999999999d3+99',
 317             'output_path': '/var/spool/cwl',
 318             'cwd': '/var/spool/cwl',
 319             'runtime_constraints': {
 320                 'API': True,
 321                 'vcpus': 1,
 322                 'ram': (1024+256)*1024*1024
 323             },
 324             'use_existing': False,
 325             'properties': {},
 326             'secret_mounts': {}
 327         }
 328
 329         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 330         stubs.api.workflows().create().execute.return_value = {
 331             "uuid": stubs.expect_workflow_uuid,
 332         }
 333         def update_mock(**kwargs):
 334             stubs.updated_uuid = kwargs.get('uuid')
 335             return mock.DEFAULT
 336         stubs.api.workflows().update.side_effect = update_mock
 337         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 338             "uuid": stubs.updated_uuid,
 339         }
 340
 341         return func(self, stubs, *args, **kwargs)
 342     return wrapped
 343
 344
 345 class TestSubmit(unittest.TestCase):
 346
 347     def setUp(self):
 348         cwltool.process._names = set()
 349         arvados_cwl.arvdocker.arv_docker_clear_cache()
 350
 351
 352     @mock.patch("time.sleep")
 353     @stubs
 354     def test_submit_invalid_runner_ram(self, stubs, tm):
 355         exited = arvados_cwl.main(
 356             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 358             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 359         self.assertEqual(exited, 1)
 360
 361
 362     @stubs
 363     def test_submit_container(self, stubs):
 364         exited = arvados_cwl.main(
 365             ["--submit", "--no-wait", "--api=containers", "--debug",
 366                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 367             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 368
 369         stubs.api.collections().create.assert_has_calls([
 370             mock.call(body=JsonDiffMatcher({
 371                 'manifest_text':
 372                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 373                 'replication_desired': None,
 374                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 375             }), ensure_unique_name=False),
 376             mock.call(body=JsonDiffMatcher({
 377                 'manifest_text':
 378                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 379                 'replication_desired': None,
 380                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 381             }), ensure_unique_name=False),
 382             ])
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         stubs.api.container_requests().create.assert_called_with(
 386             body=JsonDiffMatcher(expect_container))
 387         self.assertEqual(stubs.capture_stdout.getvalue(),
 388                          stubs.expect_container_request_uuid + '\n')
 389         self.assertEqual(exited, 0)
 390
 391
 392     @stubs
 393     def test_submit_container_tool(self, stubs):
 394         # test for issue #16139
 395         exited = arvados_cwl.main(
 396             ["--submit", "--no-wait", "--api=containers", "--debug",
 397                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 398             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 399
 400         self.assertEqual(stubs.capture_stdout.getvalue(),
 401                          stubs.expect_container_request_uuid + '\n')
 402         self.assertEqual(exited, 0)
 403
 404     @stubs
 405     def test_submit_container_no_reuse(self, stubs):
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 408                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 410
 411         expect_container = copy.deepcopy(stubs.expect_container_spec)
 412         expect_container["command"] = [
 413             'arvados-cwl-runner', '--local', '--api=containers',
 414             '--no-log-timestamps', '--disable-validate', '--disable-color',
 415             '--eval-timeout=20', '--thread-count=0',
 416             '--disable-reuse', "--collection-cache-size=256",
 417             '--debug', '--on-error=continue',
 418             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 419         expect_container["use_existing"] = False
 420
 421         stubs.api.container_requests().create.assert_called_with(
 422             body=JsonDiffMatcher(expect_container))
 423         self.assertEqual(stubs.capture_stdout.getvalue(),
 424                          stubs.expect_container_request_uuid + '\n')
 425         self.assertEqual(exited, 0)
 426
 427     @stubs
 428     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 429         exited = arvados_cwl.main(
 430             ["--submit", "--no-wait", "--api=containers", "--debug",
 431              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 432             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 433         self.assertEqual(exited, 0)
 434
 435         expect_container = copy.deepcopy(stubs.expect_container_spec)
 436         expect_container["command"] = [
 437             'arvados-cwl-runner', '--local', '--api=containers',
 438             '--no-log-timestamps', '--disable-validate', '--disable-color',
 439             '--eval-timeout=20', '--thread-count=0',
 440             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 441             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 442         expect_container["use_existing"] = False
 443         expect_container["name"] = "submit_wf_no_reuse.cwl"
 444         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 445             {
 446                 "class": "http://arvados.org/cwl#ReuseRequirement",
 447                 "enableReuse": False,
 448             },
 449         ]
 450         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 451             "arv": "http://arvados.org/cwl#",
 452             "cwltool": "http://commonwl.org/cwltool#"
 453         }
 454
 455         stubs.api.container_requests().create.assert_called_with(
 456             body=JsonDiffMatcher(expect_container))
 457         self.assertEqual(stubs.capture_stdout.getvalue(),
 458                          stubs.expect_container_request_uuid + '\n')
 459
 460
 461     @stubs
 462     def test_submit_container_on_error(self, stubs):
 463         exited = arvados_cwl.main(
 464             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 465                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 466             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 467
 468         expect_container = copy.deepcopy(stubs.expect_container_spec)
 469         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 470                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 471                                        '--eval-timeout=20', '--thread-count=0',
 472                                        '--enable-reuse', "--collection-cache-size=256",
 473                                        '--debug', '--on-error=stop',
 474                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 475
 476         stubs.api.container_requests().create.assert_called_with(
 477             body=JsonDiffMatcher(expect_container))
 478         self.assertEqual(stubs.capture_stdout.getvalue(),
 479                          stubs.expect_container_request_uuid + '\n')
 480         self.assertEqual(exited, 0)
 481
 482     @stubs
 483     def test_submit_container_output_name(self, stubs):
 484         output_name = "test_output_name"
 485
 486         exited = arvados_cwl.main(
 487             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 488                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 489             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 490
 491         expect_container = copy.deepcopy(stubs.expect_container_spec)
 492         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 493                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 494                                        '--eval-timeout=20', '--thread-count=0',
 495                                        '--enable-reuse', "--collection-cache-size=256",
 496                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 497                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 498         expect_container["output_name"] = output_name
 499
 500         stubs.api.container_requests().create.assert_called_with(
 501             body=JsonDiffMatcher(expect_container))
 502         self.assertEqual(stubs.capture_stdout.getvalue(),
 503                          stubs.expect_container_request_uuid + '\n')
 504         self.assertEqual(exited, 0)
 505
 506     @stubs
 507     def test_submit_storage_classes(self, stubs):
 508         exited = arvados_cwl.main(
 509             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 510                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 511             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 512
 513         expect_container = copy.deepcopy(stubs.expect_container_spec)
 514         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 515                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 516                                        '--eval-timeout=20', '--thread-count=0',
 517                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 518                                        "--storage-classes=foo", '--on-error=continue',
 519                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 520
 521         stubs.api.container_requests().create.assert_called_with(
 522             body=JsonDiffMatcher(expect_container))
 523         self.assertEqual(stubs.capture_stdout.getvalue(),
 524                          stubs.expect_container_request_uuid + '\n')
 525         self.assertEqual(exited, 0)
 526
 527     @stubs
 528     def test_submit_multiple_storage_classes(self, stubs):
 529         exited = arvados_cwl.main(
 530             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 531                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533
 534         expect_container = copy.deepcopy(stubs.expect_container_spec)
 535         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 536                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 537                                        '--eval-timeout=20', '--thread-count=0',
 538                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 539                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 540                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 541
 542         stubs.api.container_requests().create.assert_called_with(
 543             body=JsonDiffMatcher(expect_container))
 544         self.assertEqual(stubs.capture_stdout.getvalue(),
 545                          stubs.expect_container_request_uuid + '\n')
 546         self.assertEqual(exited, 0)
 547
 548     @mock.patch("cwltool.task_queue.TaskQueue")
 549     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 550     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 551     @stubs
 552     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 553         final_output_c = arvados.collection.Collection()
 554         make_output.return_value = ({},final_output_c)
 555
 556         def set_final_output(job_order, output_callback, runtimeContext):
 557             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 558             return []
 559         job.side_effect = set_final_output
 560
 561         exited = arvados_cwl.main(
 562             ["--debug", "--local", "--storage-classes=foo",
 563                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 564             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 565
 566         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 567         self.assertEqual(exited, 0)
 568
 569     @mock.patch("cwltool.task_queue.TaskQueue")
 570     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 571     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 572     @stubs
 573     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 574         final_output_c = arvados.collection.Collection()
 575         make_output.return_value = ({},final_output_c)
 576         stubs.api.config().get.return_value = {"default": {"Default": True}}
 577
 578         def set_final_output(job_order, output_callback, runtimeContext):
 579             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 580             return []
 581         job.side_effect = set_final_output
 582
 583         exited = arvados_cwl.main(
 584             ["--debug", "--local",
 585                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 586             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 587
 588         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 589         self.assertEqual(exited, 0)
 590
 591     @mock.patch("cwltool.task_queue.TaskQueue")
 592     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 593     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 594     @stubs
 595     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 596         final_output_c = arvados.collection.Collection()
 597         make_output.return_value = ({},final_output_c)
 598
 599         def set_final_output(job_order, output_callback, runtimeContext):
 600             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 601             return []
 602         job.side_effect = set_final_output
 603
 604         exited = arvados_cwl.main(
 605             ["--debug", "--local",
 606                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 607             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 608
 609         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 610         self.assertEqual(exited, 0)
 611
 612     @stubs
 613     def test_submit_container_output_ttl(self, stubs):
 614         exited = arvados_cwl.main(
 615             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 616                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 617             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 618
 619         expect_container = copy.deepcopy(stubs.expect_container_spec)
 620         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 621                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 622                                        '--eval-timeout=20', '--thread-count=0',
 623                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 624                                        '--on-error=continue',
 625                                        "--intermediate-output-ttl=3600",
 626                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(stubs.capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632         self.assertEqual(exited, 0)
 633
 634     @stubs
 635     def test_submit_container_trash_intermediate(self, stubs):
 636         exited = arvados_cwl.main(
 637             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 638                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 639             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 640
 641
 642         expect_container = copy.deepcopy(stubs.expect_container_spec)
 643         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 644                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 645                                        '--eval-timeout=20', '--thread-count=0',
 646                                        '--enable-reuse', "--collection-cache-size=256",
 647                                        '--debug', '--on-error=continue',
 648                                        "--trash-intermediate",
 649                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 650
 651         stubs.api.container_requests().create.assert_called_with(
 652             body=JsonDiffMatcher(expect_container))
 653         self.assertEqual(stubs.capture_stdout.getvalue(),
 654                          stubs.expect_container_request_uuid + '\n')
 655         self.assertEqual(exited, 0)
 656
 657     @stubs
 658     def test_submit_container_output_tags(self, stubs):
 659         output_tags = "tag0,tag1,tag2"
 660
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 663                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 665
 666         expect_container = copy.deepcopy(stubs.expect_container_spec)
 667         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 668                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 669                                        '--eval-timeout=20', '--thread-count=0',
 670                                        '--enable-reuse', "--collection-cache-size=256",
 671                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 672                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 673
 674         stubs.api.container_requests().create.assert_called_with(
 675             body=JsonDiffMatcher(expect_container))
 676         self.assertEqual(stubs.capture_stdout.getvalue(),
 677                          stubs.expect_container_request_uuid + '\n')
 678         self.assertEqual(exited, 0)
 679
 680     @stubs
 681     def test_submit_container_runner_ram(self, stubs):
 682         exited = arvados_cwl.main(
 683             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 684                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 685             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 686
 687         expect_container = copy.deepcopy(stubs.expect_container_spec)
 688         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 689
 690         stubs.api.container_requests().create.assert_called_with(
 691             body=JsonDiffMatcher(expect_container))
 692         self.assertEqual(stubs.capture_stdout.getvalue(),
 693                          stubs.expect_container_request_uuid + '\n')
 694         self.assertEqual(exited, 0)
 695
 696     @mock.patch("arvados.collection.CollectionReader")
 697     @mock.patch("time.sleep")
 698     @stubs
 699     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 700         collectionReader().exists.return_value = True
 701         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 702         exited = arvados_cwl.main(
 703             ["--submit", "--no-wait", "--api=containers", "--debug",
 704              "tests/wf/submit_keepref_wf.cwl"],
 705             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 706         self.assertEqual(exited, 0)
 707
 708     @mock.patch("arvados.collection.CollectionReader")
 709     @mock.patch("time.sleep")
 710     @stubs
 711     def test_submit_keepref(self, stubs, tm, reader):
 712         with open("tests/wf/expect_arvworkflow.cwl") as f:
 713             reader().open().__enter__().read.return_value = f.read()
 714
 715         exited = arvados_cwl.main(
 716             ["--submit", "--no-wait", "--api=containers", "--debug",
 717              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 718             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 719
 720         expect_container = {
 721             'priority': 500,
 722             'mounts': {
 723                 '/var/spool/cwl': {
 724                     'writable': True,
 725                     'kind': 'collection'
 726                 },
 727                 'stdout': {
 728                     'path': '/var/spool/cwl/cwl.output.json',
 729                     'kind': 'file'
 730                 },
 731                 '/var/lib/cwl/workflow': {
 732                     'portable_data_hash': '99999999999999999999999999999994+99',
 733                     'kind': 'collection'
 734                 },
 735                 '/var/lib/cwl/cwl.input.json': {
 736                     'content': {
 737                         'x': 'XxX'
 738                     },
 739                     'kind': 'json'
 740                 }
 741             }, 'state': 'Committed',
 742             'output_path': '/var/spool/cwl',
 743             'name': 'expect_arvworkflow.cwl#main',
 744             'container_image': '999999999999999999999999999999d3+99',
 745             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 746                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 747                         '--eval-timeout=20', '--thread-count=0',
 748                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 749                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 750             'cwd': '/var/spool/cwl',
 751             'runtime_constraints': {
 752                 'API': True,
 753                 'vcpus': 1,
 754                 'ram': 1342177280
 755             },
 756             'use_existing': False,
 757             'properties': {},
 758             'secret_mounts': {}
 759         }
 760
 761         stubs.api.container_requests().create.assert_called_with(
 762             body=JsonDiffMatcher(expect_container))
 763         self.assertEqual(stubs.capture_stdout.getvalue(),
 764                          stubs.expect_container_request_uuid + '\n')
 765         self.assertEqual(exited, 0)
 766
 767     @mock.patch("time.sleep")
 768     @stubs
 769     def test_submit_arvworkflow(self, stubs, tm):
 770         with open("tests/wf/expect_arvworkflow.cwl") as f:
 771             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 772
 773         exited = arvados_cwl.main(
 774             ["--submit", "--no-wait", "--api=containers", "--debug",
 775              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 776             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 777
 778         expect_container = {
 779             'priority': 500,
 780             'mounts': {
 781                 '/var/spool/cwl': {
 782                     'writable': True,
 783                     'kind': 'collection'
 784                 },
 785                 'stdout': {
 786                     'path': '/var/spool/cwl/cwl.output.json',
 787                     'kind': 'file'
 788                 },
 789                 '/var/lib/cwl/workflow.json': {
 790                     'kind': 'json',
 791                     'content': {
 792                         'cwlVersion': 'v1.0',
 793                         '$graph': [
 794                             {
 795                                 'id': '#main',
 796                                 'inputs': [
 797                                     {'type': 'string', 'id': '#main/x'}
 798                                 ],
 799                                 'steps': [
 800                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 801                                      'run': '#submit_tool.cwl',
 802                                      'id': '#main/step1',
 803                                      'out': []}
 804                                 ],
 805                                 'class': 'Workflow',
 806                                 'outputs': []
 807                             },
 808                             {
 809                                 'inputs': [
 810                                     {
 811                                         'inputBinding': {'position': 1},
 812                                         'type': 'string',
 813                                         'id': '#submit_tool.cwl/x'}
 814                                 ],
 815                                 'requirements': [
 816                                     {
 817                                         'dockerPull': 'debian:buster-slim',
 818                                         'class': 'DockerRequirement',
 819                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 820                                     }
 821                                 ],
 822                                 'id': '#submit_tool.cwl',
 823                                 'outputs': [],
 824                                 'baseCommand': 'cat',
 825                                 'class': 'CommandLineTool'
 826                             }
 827                         ]
 828                     }
 829                 },
 830                 '/var/lib/cwl/cwl.input.json': {
 831                     'content': {
 832                         'x': 'XxX'
 833                     },
 834                     'kind': 'json'
 835                 }
 836             }, 'state': 'Committed',
 837             'output_path': '/var/spool/cwl',
 838             'name': 'a test workflow',
 839             'container_image': "999999999999999999999999999999d3+99",
 840             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 841                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 842                         '--eval-timeout=20', '--thread-count=0',
 843                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 844                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 845             'cwd': '/var/spool/cwl',
 846             'runtime_constraints': {
 847                 'API': True,
 848                 'vcpus': 1,
 849                 'ram': 1342177280
 850             },
 851             'use_existing': False,
 852             'properties': {
 853                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 854             },
 855             'secret_mounts': {}
 856         }
 857
 858         stubs.api.container_requests().create.assert_called_with(
 859             body=JsonDiffMatcher(expect_container))
 860         self.assertEqual(stubs.capture_stdout.getvalue(),
 861                          stubs.expect_container_request_uuid + '\n')
 862         self.assertEqual(exited, 0)
 863
 864     @stubs
 865     def test_submit_container_name(self, stubs):
 866         exited = arvados_cwl.main(
 867             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 868                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 869             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 870
 871         expect_container = copy.deepcopy(stubs.expect_container_spec)
 872         expect_container["name"] = "hello container 123"
 873
 874         stubs.api.container_requests().create.assert_called_with(
 875             body=JsonDiffMatcher(expect_container))
 876         self.assertEqual(stubs.capture_stdout.getvalue(),
 877                          stubs.expect_container_request_uuid + '\n')
 878         self.assertEqual(exited, 0)
 879
 880     @stubs
 881     def test_submit_missing_input(self, stubs):
 882         exited = arvados_cwl.main(
 883             ["--submit", "--no-wait", "--api=containers", "--debug",
 884              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 885             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 886         self.assertEqual(exited, 0)
 887
 888         exited = arvados_cwl.main(
 889             ["--submit", "--no-wait", "--api=containers", "--debug",
 890              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 891             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 892         self.assertEqual(exited, 1)
 893
 894     @stubs
 895     def test_submit_container_project(self, stubs):
 896         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 897         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 898         exited = arvados_cwl.main(
 899             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 900                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 901             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 902
 903         expect_container = copy.deepcopy(stubs.expect_container_spec)
 904         expect_container["owner_uuid"] = project_uuid
 905         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 906                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 907                                        "--eval-timeout=20", "--thread-count=0",
 908                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 909                                        '--on-error=continue',
 910                                        '--project-uuid='+project_uuid,
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(stubs.capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917         self.assertEqual(exited, 0)
 918
 919     @stubs
 920     def test_submit_container_eval_timeout(self, stubs):
 921         exited = arvados_cwl.main(
 922             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 923                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 924             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 925
 926         expect_container = copy.deepcopy(stubs.expect_container_spec)
 927         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 928                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 929                                        '--eval-timeout=60.0', '--thread-count=0',
 930                                        '--enable-reuse', "--collection-cache-size=256",
 931                                        '--debug', '--on-error=continue',
 932                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 933
 934         stubs.api.container_requests().create.assert_called_with(
 935             body=JsonDiffMatcher(expect_container))
 936         self.assertEqual(stubs.capture_stdout.getvalue(),
 937                          stubs.expect_container_request_uuid + '\n')
 938         self.assertEqual(exited, 0)
 939
 940     @stubs
 941     def test_submit_container_collection_cache(self, stubs):
 942         exited = arvados_cwl.main(
 943             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 944                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 945             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 946
 947         expect_container = copy.deepcopy(stubs.expect_container_spec)
 948         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 949                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 950                                        '--eval-timeout=20', '--thread-count=0',
 951                                        '--enable-reuse', "--collection-cache-size=500",
 952                                        '--debug', '--on-error=continue',
 953                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 954         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 955
 956         stubs.api.container_requests().create.assert_called_with(
 957             body=JsonDiffMatcher(expect_container))
 958         self.assertEqual(stubs.capture_stdout.getvalue(),
 959                          stubs.expect_container_request_uuid + '\n')
 960         self.assertEqual(exited, 0)
 961
 962     @stubs
 963     def test_submit_container_thread_count(self, stubs):
 964         exited = arvados_cwl.main(
 965             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 966                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 967             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 968
 969         expect_container = copy.deepcopy(stubs.expect_container_spec)
 970         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 971                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 972                                        '--eval-timeout=20', '--thread-count=20',
 973                                        '--enable-reuse', "--collection-cache-size=256",
 974                                        '--debug', '--on-error=continue',
 975                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 976
 977         stubs.api.container_requests().create.assert_called_with(
 978             body=JsonDiffMatcher(expect_container))
 979         self.assertEqual(stubs.capture_stdout.getvalue(),
 980                          stubs.expect_container_request_uuid + '\n')
 981         self.assertEqual(exited, 0)
 982
 983     @stubs
 984     def test_submit_container_runner_image(self, stubs):
 985         exited = arvados_cwl.main(
 986             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 987                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 988             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 989
 990         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 991
 992         expect_container = copy.deepcopy(stubs.expect_container_spec)
 993         stubs.api.container_requests().create.assert_called_with(
 994             body=JsonDiffMatcher(expect_container))
 995         self.assertEqual(stubs.capture_stdout.getvalue(),
 996                          stubs.expect_container_request_uuid + '\n')
 997         self.assertEqual(exited, 0)
 998
 999     @stubs
1000     def test_submit_priority(self, stubs):
1001         exited = arvados_cwl.main(
1002             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1003                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1004             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1005
1006         stubs.expect_container_spec["priority"] = 669
1007
1008         expect_container = copy.deepcopy(stubs.expect_container_spec)
1009         stubs.api.container_requests().create.assert_called_with(
1010             body=JsonDiffMatcher(expect_container))
1011         self.assertEqual(stubs.capture_stdout.getvalue(),
1012                          stubs.expect_container_request_uuid + '\n')
1013         self.assertEqual(exited, 0)
1014
1015     @stubs
1016     def test_submit_wf_runner_resources(self, stubs):
1017         exited = arvados_cwl.main(
1018             ["--submit", "--no-wait", "--api=containers", "--debug",
1019                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1020             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1021
1022         expect_container = copy.deepcopy(stubs.expect_container_spec)
1023         expect_container["runtime_constraints"] = {
1024             "API": True,
1025             "vcpus": 2,
1026             "ram": (2000+512) * 2**20
1027         }
1028         expect_container["name"] = "submit_wf_runner_resources.cwl"
1029         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1030             {
1031                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1032                 "coresMin": 2,
1033                 "ramMin": 2000,
1034                 "keep_cache": 512
1035             }
1036         ]
1037         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1038             "arv": "http://arvados.org/cwl#",
1039         }
1040         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1041                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1042                         '--eval-timeout=20', '--thread-count=0',
1043                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1044                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1045
1046         stubs.api.container_requests().create.assert_called_with(
1047             body=JsonDiffMatcher(expect_container))
1048         self.assertEqual(stubs.capture_stdout.getvalue(),
1049                          stubs.expect_container_request_uuid + '\n')
1050         self.assertEqual(exited, 0)
1051
1052     def tearDown(self):
1053         arvados_cwl.arvdocker.arv_docker_clear_cache()
1054
1055     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1056     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1057     @mock.patch("arvados.api")
1058     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1059         arvados_cwl.arvdocker.arv_docker_clear_cache()
1060
1061         arvrunner = mock.MagicMock()
1062         arvrunner.project_uuid = ""
1063         api.return_value = mock.MagicMock()
1064         arvrunner.api = api.return_value
1065         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1066                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1067                                                                         "link_class": "docker_image_repo+tag",
1068                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1069                                                                         "owner_uuid": "",
1070                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1071                                                             {"items": [{"created_at": "",
1072                                                                         "head_uuid": "",
1073                                                                         "link_class": "docker_image_hash",
1074                                                                         "name": "123456",
1075                                                                         "owner_uuid": "",
1076                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1077         )
1078         find_one_image_hash.return_value = "123456"
1079
1080         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1081                                                                               "owner_uuid": "",
1082                                                                               "manifest_text": "",
1083                                                                               "properties": ""
1084                                                                           }], "items_available": 1, "offset": 0},)
1085         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1086         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1087                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1088         self.assertEqual("9999999999999999999999999999999b+99",
1089                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1090
1091
1092     @stubs
1093     def test_match_local_docker(self, stubs):
1094         stubs.docker_images["debian:buster-slim"] = [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {"dockerhash": "456"}),
1095                                                      ("zzzzz-4zz18-zzzzzzzzzzzzzd5", {"dockerhash": "123"})]
1096
1097         exited = arvados_cwl.main(
1098             ["--submit", "--no-wait", "--api=containers", "--debug",
1099                 "tests/tool/submit_tool.cwl", "tests/submit_test_job.json"],
1100             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1101
1102         expect_container = {
1103         }
1104
1105         stubs.api.container_requests().create.assert_called_with(
1106             body=JsonDiffMatcher(expect_container))
1107         self.assertEqual(stubs.capture_stdout.getvalue(),
1108                          stubs.expect_container_request_uuid + '\n')
1109         self.assertEqual(exited, 0)
1110
1111     @stubs
1112     def test_submit_secrets(self, stubs):
1113         exited = arvados_cwl.main(
1114             ["--submit", "--no-wait", "--api=containers", "--debug",
1115                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1116             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1117
1118         expect_container = {
1119             "command": [
1120                 "arvados-cwl-runner",
1121                 "--local",
1122                 "--api=containers",
1123                 "--no-log-timestamps",
1124                 "--disable-validate",
1125                 "--disable-color",
1126                 "--eval-timeout=20",
1127                 '--thread-count=0',
1128                 "--enable-reuse",
1129                 "--collection-cache-size=256",
1130                 '--debug',
1131                 "--on-error=continue",
1132                 "/var/lib/cwl/workflow.json#main",
1133                 "/var/lib/cwl/cwl.input.json"
1134             ],
1135             "container_image": "999999999999999999999999999999d3+99",
1136             "cwd": "/var/spool/cwl",
1137             "mounts": {
1138                 "/var/lib/cwl/cwl.input.json": {
1139                     "content": {
1140                         "pw": {
1141                             "$include": "/secrets/s0"
1142                         }
1143                     },
1144                     "kind": "json"
1145                 },
1146                 "/var/lib/cwl/workflow.json": {
1147                     "content": {
1148                         "$graph": [
1149                             {
1150                                 "arguments": [
1151                                     "md5sum",
1152                                     "example.conf"
1153                                 ],
1154                                 "class": "CommandLineTool",
1155                                 "hints": [
1156                                     {
1157                                         "class": "http://commonwl.org/cwltool#Secrets",
1158                                         "secrets": [
1159                                             "#secret_job.cwl/pw"
1160                                         ]
1161                                     }
1162                                 ],
1163                                 "id": "#secret_job.cwl",
1164                                 "inputs": [
1165                                     {
1166                                         "id": "#secret_job.cwl/pw",
1167                                         "type": "string"
1168                                     }
1169                                 ],
1170                                 "outputs": [
1171                                     {
1172                                         "id": "#secret_job.cwl/out",
1173                                         "type": "File",
1174                                         "outputBinding": {
1175                                               "glob": "hashed_example.txt"
1176                                         }
1177                                     }
1178                                 ],
1179                                 "stdout": "hashed_example.txt",
1180                                 "requirements": [
1181                                     {
1182                                         "class": "InitialWorkDirRequirement",
1183                                         "listing": [
1184                                             {
1185                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1186                                                 "entryname": "example.conf"
1187                                             }
1188                                         ]
1189                                     }
1190                                 ]
1191                             },
1192                             {
1193                                 "class": "Workflow",
1194                                 "hints": [
1195                                     {
1196                                         "class": "DockerRequirement",
1197                                         "dockerPull": "debian:buster-slim",
1198                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1199                                     },
1200                                     {
1201                                         "class": "http://commonwl.org/cwltool#Secrets",
1202                                         "secrets": [
1203                                             "#main/pw"
1204                                         ]
1205                                     }
1206                                 ],
1207                                 "id": "#main",
1208                                 "inputs": [
1209                                     {
1210                                         "id": "#main/pw",
1211                                         "type": "string"
1212                                     }
1213                                 ],
1214                                 "outputs": [
1215                                     {
1216                                         "id": "#main/out",
1217                                         "outputSource": "#main/step1/out",
1218                                         "type": "File"
1219                                     }
1220                                 ],
1221                                 "steps": [
1222                                     {
1223                                         "id": "#main/step1",
1224                                         "in": [
1225                                             {
1226                                                 "id": "#main/step1/pw",
1227                                                 "source": "#main/pw"
1228                                             }
1229                                         ],
1230                                         "out": [
1231                                             "#main/step1/out"
1232                                         ],
1233                                         "run": "#secret_job.cwl"
1234                                     }
1235                                 ]
1236                             }
1237                         ],
1238                         "$namespaces": {
1239                             "cwltool": "http://commonwl.org/cwltool#"
1240                         },
1241                         "cwlVersion": "v1.0"
1242                     },
1243                     "kind": "json"
1244                 },
1245                 "/var/spool/cwl": {
1246                     "kind": "collection",
1247                     "writable": True
1248                 },
1249                 "stdout": {
1250                     "kind": "file",
1251                     "path": "/var/spool/cwl/cwl.output.json"
1252                 }
1253             },
1254             "name": "secret_wf.cwl",
1255             "output_path": "/var/spool/cwl",
1256             "priority": 500,
1257             "properties": {},
1258             "runtime_constraints": {
1259                 "API": True,
1260                 "ram": 1342177280,
1261                 "vcpus": 1
1262             },
1263             "secret_mounts": {
1264                 "/secrets/s0": {
1265                     "content": "blorp",
1266                     "kind": "text"
1267                 }
1268             },
1269             "state": "Committed",
1270             "use_existing": False
1271         }
1272
1273         stubs.api.container_requests().create.assert_called_with(
1274             body=JsonDiffMatcher(expect_container))
1275         self.assertEqual(stubs.capture_stdout.getvalue(),
1276                          stubs.expect_container_request_uuid + '\n')
1277         self.assertEqual(exited, 0)
1278
1279     @stubs
1280     def test_submit_request_uuid(self, stubs):
1281         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1282         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1283
1284         stubs.api.container_requests().update().execute.return_value = {
1285             "uuid": stubs.expect_container_request_uuid,
1286             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1287             "state": "Queued"
1288         }
1289
1290         exited = arvados_cwl.main(
1291             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1292                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1293             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1294
1295         stubs.api.container_requests().update.assert_called_with(
1296             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1297         self.assertEqual(stubs.capture_stdout.getvalue(),
1298                          stubs.expect_container_request_uuid + '\n')
1299         self.assertEqual(exited, 0)
1300
1301     @stubs
1302     def test_submit_container_cluster_id(self, stubs):
1303         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1304
1305         exited = arvados_cwl.main(
1306             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1307                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1308             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1309
1310         expect_container = copy.deepcopy(stubs.expect_container_spec)
1311
1312         stubs.api.container_requests().create.assert_called_with(
1313             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1314         self.assertEqual(stubs.capture_stdout.getvalue(),
1315                          stubs.expect_container_request_uuid + '\n')
1316         self.assertEqual(exited, 0)
1317
1318     @stubs
1319     def test_submit_validate_cluster_id(self, stubs):
1320         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1321         exited = arvados_cwl.main(
1322             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1323              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1324             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1325         self.assertEqual(exited, 1)
1326
1327     @stubs
1328     def test_submit_validate_project_uuid(self, stubs):
1329         # Fails with bad cluster prefix
1330         exited = arvados_cwl.main(
1331             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1332              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1333             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1334         self.assertEqual(exited, 1)
1335
1336         # Project lookup fails
1337         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1338         exited = arvados_cwl.main(
1339             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1340              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1341             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1342         self.assertEqual(exited, 1)
1343
1344         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1345         exited = arvados_cwl.main(
1346             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1347              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1348             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1349         self.assertEqual(exited, 0)
1350
1351
1352     @mock.patch("arvados.collection.CollectionReader")
1353     @stubs
1354     def test_submit_uuid_inputs(self, stubs, collectionReader):
1355         collectionReader().exists.return_value = True
1356         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1357         def list_side_effect(**kwargs):
1358             m = mock.MagicMock()
1359             if "count" in kwargs:
1360                 m.execute.return_value = {"items": [
1361                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1362                 ]}
1363             else:
1364                 m.execute.return_value = {"items": []}
1365             return m
1366         stubs.api.collections().list.side_effect = list_side_effect
1367
1368         exited = arvados_cwl.main(
1369             ["--submit", "--no-wait", "--api=containers", "--debug",
1370                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1371             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1372
1373         expect_container = copy.deepcopy(stubs.expect_container_spec)
1374         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1375         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1376         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1377
1378         stubs.api.collections().list.assert_has_calls([
1379             mock.call(count='none',
1380                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1381                       select=['uuid', 'portable_data_hash'])])
1382         stubs.api.container_requests().create.assert_called_with(
1383             body=JsonDiffMatcher(expect_container))
1384         self.assertEqual(stubs.capture_stdout.getvalue(),
1385                          stubs.expect_container_request_uuid + '\n')
1386         self.assertEqual(exited, 0)
1387
1388     @stubs
1389     def test_submit_mismatched_uuid_inputs(self, stubs):
1390         def list_side_effect(**kwargs):
1391             m = mock.MagicMock()
1392             if "count" in kwargs:
1393                 m.execute.return_value = {"items": [
1394                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1395                 ]}
1396             else:
1397                 m.execute.return_value = {"items": []}
1398             return m
1399         stubs.api.collections().list.side_effect = list_side_effect
1400
1401         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1402             capture_stderr = StringIO()
1403             cwltool_logger = logging.getLogger('cwltool')
1404             stderr_logger = logging.StreamHandler(capture_stderr)
1405             cwltool_logger.addHandler(stderr_logger)
1406
1407             try:
1408                 exited = arvados_cwl.main(
1409                     ["--submit", "--no-wait", "--api=containers", "--debug",
1410                         "tests/wf/submit_wf.cwl", infile],
1411                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1412
1413                 self.assertEqual(exited, 1)
1414                 self.assertRegex(
1415                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1416                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1417             finally:
1418                 cwltool_logger.removeHandler(stderr_logger)
1419
1420     @mock.patch("arvados.collection.CollectionReader")
1421     @stubs
1422     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1423         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1424         capture_stderr = StringIO()
1425
1426         cwltool_logger = logging.getLogger('cwltool')
1427         stderr_logger = logging.StreamHandler(capture_stderr)
1428         cwltool_logger.addHandler(stderr_logger)
1429
1430         exited = arvados_cwl.main(
1431             ["--submit", "--no-wait", "--api=containers", "--debug",
1432                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1433             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1434
1435         try:
1436             self.assertEqual(exited, 1)
1437             self.assertRegex(
1438                 capture_stderr.getvalue(),
1439                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1440         finally:
1441             cwltool_logger.removeHandler(stderr_logger)
1442
1443     @stubs
1444     def test_submit_set_process_properties(self, stubs):
1445         exited = arvados_cwl.main(
1446             ["--submit", "--no-wait", "--api=containers", "--debug",
1447                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1448             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1449
1450         expect_container = copy.deepcopy(stubs.expect_container_spec)
1451         expect_container["name"] = "submit_wf_process_properties.cwl"
1452         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1453             {
1454                 "class": "http://arvados.org/cwl#ProcessProperties",
1455                 "processProperties": [
1456                     {"propertyName": "baz",
1457                      "propertyValue": "$(inputs.x.basename)"},
1458                     {"propertyName": "foo",
1459                      "propertyValue": "bar"},
1460                     {"propertyName": "quux",
1461                      "propertyValue": {
1462                          "q1": 1,
1463                          "q2": 2
1464                      }
1465                     }
1466                 ],
1467             }
1468         ]
1469         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1470             "arv": "http://arvados.org/cwl#"
1471         }
1472
1473         expect_container["properties"] = {
1474             "baz": "blorp.txt",
1475             "foo": "bar",
1476             "quux": {
1477                 "q1": 1,
1478                 "q2": 2
1479             }
1480         }
1481
1482         stubs.api.container_requests().create.assert_called_with(
1483             body=JsonDiffMatcher(expect_container))
1484         self.assertEqual(stubs.capture_stdout.getvalue(),
1485                          stubs.expect_container_request_uuid + '\n')
1486         self.assertEqual(exited, 0)
1487
1488
1489 class TestCreateWorkflow(unittest.TestCase):
1490     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1491     expect_workflow = StripYAMLComments(
1492         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1493
1494     def setUp(self):
1495         cwltool.process._names = set()
1496         arvados_cwl.arvdocker.arv_docker_clear_cache()
1497
1498     @stubs
1499     def test_create(self, stubs):
1500         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1501         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1502
1503         exited = arvados_cwl.main(
1504             ["--create-workflow", "--debug",
1505              "--api=containers",
1506              "--project-uuid", project_uuid,
1507              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1508             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1509
1510         stubs.api.pipeline_templates().create.refute_called()
1511         stubs.api.container_requests().create.refute_called()
1512
1513         body = {
1514             "workflow": {
1515                 "owner_uuid": project_uuid,
1516                 "name": "submit_wf.cwl",
1517                 "description": "",
1518                 "definition": self.expect_workflow,
1519             }
1520         }
1521         stubs.api.workflows().create.assert_called_with(
1522             body=JsonDiffMatcher(body))
1523
1524         self.assertEqual(stubs.capture_stdout.getvalue(),
1525                          stubs.expect_workflow_uuid + '\n')
1526         self.assertEqual(exited, 0)
1527
1528     @stubs
1529     def test_create_name(self, stubs):
1530         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1531         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1532
1533         exited = arvados_cwl.main(
1534             ["--create-workflow", "--debug",
1535              "--api=containers",
1536              "--project-uuid", project_uuid,
1537              "--name", "testing 123",
1538              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1539             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1540
1541         stubs.api.pipeline_templates().create.refute_called()
1542         stubs.api.container_requests().create.refute_called()
1543
1544         body = {
1545             "workflow": {
1546                 "owner_uuid": project_uuid,
1547                 "name": "testing 123",
1548                 "description": "",
1549                 "definition": self.expect_workflow,
1550             }
1551         }
1552         stubs.api.workflows().create.assert_called_with(
1553             body=JsonDiffMatcher(body))
1554
1555         self.assertEqual(stubs.capture_stdout.getvalue(),
1556                          stubs.expect_workflow_uuid + '\n')
1557         self.assertEqual(exited, 0)
1558
1559
1560     @stubs
1561     def test_update(self, stubs):
1562         exited = arvados_cwl.main(
1563             ["--update-workflow", self.existing_workflow_uuid,
1564              "--debug",
1565              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1566             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1567
1568         body = {
1569             "workflow": {
1570                 "name": "submit_wf.cwl",
1571                 "description": "",
1572                 "definition": self.expect_workflow,
1573             }
1574         }
1575         stubs.api.workflows().update.assert_called_with(
1576             uuid=self.existing_workflow_uuid,
1577             body=JsonDiffMatcher(body))
1578         self.assertEqual(stubs.capture_stdout.getvalue(),
1579                          self.existing_workflow_uuid + '\n')
1580         self.assertEqual(exited, 0)
1581
1582     @stubs
1583     def test_update_name(self, stubs):
1584         exited = arvados_cwl.main(
1585             ["--update-workflow", self.existing_workflow_uuid,
1586              "--debug", "--name", "testing 123",
1587              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1588             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1589
1590         body = {
1591             "workflow": {
1592                 "name": "testing 123",
1593                 "description": "",
1594                 "definition": self.expect_workflow,
1595             }
1596         }
1597         stubs.api.workflows().update.assert_called_with(
1598             uuid=self.existing_workflow_uuid,
1599             body=JsonDiffMatcher(body))
1600         self.assertEqual(stubs.capture_stdout.getvalue(),
1601                          self.existing_workflow_uuid + '\n')
1602         self.assertEqual(exited, 0)
1603
1604     @stubs
1605     def test_create_collection_per_tool(self, stubs):
1606         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1607         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1608
1609         exited = arvados_cwl.main(
1610             ["--create-workflow", "--debug",
1611              "--api=containers",
1612              "--project-uuid", project_uuid,
1613              "tests/collection_per_tool/collection_per_tool.cwl"],
1614             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1615
1616         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1617         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1618
1619         body = {
1620             "workflow": {
1621                 "owner_uuid": project_uuid,
1622                 "name": "collection_per_tool.cwl",
1623                 "description": "",
1624                 "definition": expect_workflow,
1625             }
1626         }
1627         stubs.api.workflows().create.assert_called_with(
1628             body=JsonDiffMatcher(body))
1629
1630         self.assertEqual(stubs.capture_stdout.getvalue(),
1631                          stubs.expect_workflow_uuid + '\n')
1632         self.assertEqual(exited, 0)
1633
1634     @stubs
1635     def test_create_with_imports(self, stubs):
1636         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1637         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1638
1639         exited = arvados_cwl.main(
1640             ["--create-workflow", "--debug",
1641              "--api=containers",
1642              "--project-uuid", project_uuid,
1643              "tests/wf/feddemo/feddemo.cwl"],
1644             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1645
1646         stubs.api.pipeline_templates().create.refute_called()
1647         stubs.api.container_requests().create.refute_called()
1648
1649         self.assertEqual(stubs.capture_stdout.getvalue(),
1650                          stubs.expect_workflow_uuid + '\n')
1651         self.assertEqual(exited, 0)
1652
1653     @stubs
1654     def test_create_with_no_input(self, stubs):
1655         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1656         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1657
1658         exited = arvados_cwl.main(
1659             ["--create-workflow", "--debug",
1660              "--api=containers",
1661              "--project-uuid", project_uuid,
1662              "tests/wf/revsort/revsort.cwl"],
1663             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1664
1665         stubs.api.pipeline_templates().create.refute_called()
1666         stubs.api.container_requests().create.refute_called()
1667
1668         self.assertEqual(stubs.capture_stdout.getvalue(),
1669                          stubs.expect_workflow_uuid + '\n')
1670         self.assertEqual(exited, 0)