sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  71             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90         stubs.api._rootDesc["revision"] = "20210628"
  91
  92         stubs.api.users().current().execute.return_value = {
  93             "uuid": stubs.fake_user_uuid,
  94         }
  95         stubs.api.collections().list().execute.return_value = {"items": []}
  96         stubs.api.containers().current().execute.return_value = {
  97             "uuid": stubs.fake_container_uuid,
  98         }
  99         stubs.api.config()["StorageClasses"].items.return_value = {
 100             "default": {
 101                 "Default": True
 102             }
 103         }.items()
 104
 105         class CollectionExecute(object):
 106             def __init__(self, exe):
 107                 self.exe = exe
 108             def execute(self, num_retries=None):
 109                 return self.exe
 110
 111         def collection_createstub(created_collections, body, ensure_unique_name=None):
 112             mt = body["manifest_text"].encode('utf-8')
 113             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 114             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 115             created_collections[uuid] = {
 116                 "uuid": uuid,
 117                 "portable_data_hash": pdh,
 118                 "manifest_text": mt.decode('utf-8')
 119             }
 120             return CollectionExecute(created_collections[uuid])
 121
 122         def collection_getstub(created_collections, uuid):
 123             for v in viewvalues(created_collections):
 124                 if uuid in (v["uuid"], v["portable_data_hash"]):
 125                     return CollectionExecute(v)
 126
 127         created_collections = {
 128             "99999999999999999999999999999998+99": {
 129                 "uuid": "",
 130                 "portable_data_hash": "99999999999999999999999999999998+99",
 131                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 132             },
 133             "99999999999999999999999999999997+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999997+99",
 136                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999994+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999994+99",
 141                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 142             },
 143             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 144                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 145                 "portable_data_hash": "999999999999999999999999999999d3+99",
 146                 "manifest_text": ""
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 150                 "portable_data_hash": "999999999999999999999999999999d4+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 155                 "portable_data_hash": "999999999999999999999999999999d5+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 160                 "portable_data_hash": "999999999999999999999999999999d6+99",
 161                 "manifest_text": ""
 162             }
 163         }
 164         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 165         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 166
 167         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 168         stubs.api.jobs().create().execute.return_value = {
 169             "uuid": stubs.expect_job_uuid,
 170             "state": "Queued",
 171         }
 172
 173         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 174         stubs.api.container_requests().create().execute.return_value = {
 175             "uuid": stubs.expect_container_request_uuid,
 176             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 177             "state": "Queued"
 178         }
 179
 180         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 181         stubs.api.pipeline_templates().create().execute.return_value = {
 182             "uuid": stubs.expect_pipeline_template_uuid,
 183         }
 184         stubs.expect_job_spec = {
 185             'runtime_constraints': {
 186                 'docker_image': '999999999999999999999999999999d3+99',
 187                 'min_ram_mb_per_node': 1024
 188             },
 189             'script_parameters': {
 190                 'x': {
 191                     'basename': 'blorp.txt',
 192                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 193                     'class': 'File'
 194                 },
 195                 'y': {
 196                     'basename': '99999999999999999999999999999998+99',
 197                     'location': 'keep:99999999999999999999999999999998+99',
 198                     'class': 'Directory'
 199                 },
 200                 'z': {
 201                     'basename': 'anonymous',
 202                     "listing": [{
 203                         "basename": "renamed.txt",
 204                         "class": "File",
 205                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 206                         "size": 0
 207                     }],
 208                     'class': 'Directory'
 209                 },
 210                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 211             },
 212             'repository': 'arvados',
 213             'script_version': 'master',
 214             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 215             'script': 'cwl-runner'
 216         }
 217         stubs.pipeline_component = stubs.expect_job_spec.copy()
 218         stubs.expect_pipeline_instance = {
 219             'name': 'submit_wf.cwl',
 220             'state': 'RunningOnServer',
 221             'owner_uuid': None,
 222             "components": {
 223                 "cwl-runner": {
 224                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 225                     'script_parameters': {
 226                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 227                         'x': {"value": {
 228                             'basename': 'blorp.txt',
 229                             'class': 'File',
 230                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 231                             "size": 16
 232                         }},
 233                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 234                               'listing': [
 235                                   {
 236                                       'basename': 'renamed.txt',
 237                                       'class': 'File', 'location':
 238                                       'keep:99999999999999999999999999999998+99/file1.txt',
 239                                       'size': 0
 240                                   }
 241                               ]}},
 242                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 243                         'arv:debug': True,
 244                         'arv:enable_reuse': True,
 245                         'arv:on_error': 'continue'
 246                     },
 247                     'repository': 'arvados',
 248                     'script_version': 'master',
 249                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 250                     'script': 'cwl-runner',
 251                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 252                 }
 253             }
 254         }
 255         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 256         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 257         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 258         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 259         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 260             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 261             "state": "Queued"
 262         }
 263         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 264         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 265
 266         with open("tests/wf/submit_wf_packed.cwl") as f:
 267             expect_packed_workflow = yaml.round_trip_load(f)
 268
 269         stubs.expect_container_spec = {
 270             'priority': 500,
 271             'mounts': {
 272                 '/var/spool/cwl': {
 273                     'writable': True,
 274                     'kind': 'collection'
 275                 },
 276                 '/var/lib/cwl/workflow.json': {
 277                     'content': expect_packed_workflow,
 278                     'kind': 'json'
 279                 },
 280                 'stdout': {
 281                     'path': '/var/spool/cwl/cwl.output.json',
 282                     'kind': 'file'
 283                 },
 284                 '/var/lib/cwl/cwl.input.json': {
 285                     'kind': 'json',
 286                     'content': {
 287                         'y': {
 288                             'basename': '99999999999999999999999999999998+99',
 289                             'location': 'keep:99999999999999999999999999999998+99',
 290                             'class': 'Directory'},
 291                         'x': {
 292                             'basename': u'blorp.txt',
 293                             'class': 'File',
 294                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 295                             "size": 16
 296                         },
 297                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 298                             {'basename': 'renamed.txt',
 299                              'class': 'File',
 300                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 301                              'size': 0
 302                             }
 303                         ]}
 304                     },
 305                     'kind': 'json'
 306                 }
 307             },
 308             'secret_mounts': {},
 309             'state': 'Committed',
 310             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 311                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 312                         '--eval-timeout=20', '--thread-count=0',
 313                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 314                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 315             'name': 'submit_wf.cwl',
 316             'container_image': '999999999999999999999999999999d3+99',
 317             'output_path': '/var/spool/cwl',
 318             'cwd': '/var/spool/cwl',
 319             'runtime_constraints': {
 320                 'API': True,
 321                 'vcpus': 1,
 322                 'ram': (1024+256)*1024*1024
 323             },
 324             'use_existing': False,
 325             'properties': {},
 326             'secret_mounts': {}
 327         }
 328
 329         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 330         stubs.api.workflows().create().execute.return_value = {
 331             "uuid": stubs.expect_workflow_uuid,
 332         }
 333         def update_mock(**kwargs):
 334             stubs.updated_uuid = kwargs.get('uuid')
 335             return mock.DEFAULT
 336         stubs.api.workflows().update.side_effect = update_mock
 337         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 338             "uuid": stubs.updated_uuid,
 339         }
 340
 341         return func(self, stubs, *args, **kwargs)
 342     return wrapped
 343
 344
 345 class TestSubmit(unittest.TestCase):
 346
 347     def setUp(self):
 348         cwltool.process._names = set()
 349         arvados_cwl.arvdocker.arv_docker_clear_cache()
 350
 351
 352     @mock.patch("time.sleep")
 353     @stubs
 354     def test_submit_invalid_runner_ram(self, stubs, tm):
 355         exited = arvados_cwl.main(
 356             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 358             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 359         self.assertEqual(exited, 1)
 360
 361
 362     @stubs
 363     def test_submit_container(self, stubs):
 364         exited = arvados_cwl.main(
 365             ["--submit", "--no-wait", "--api=containers", "--debug",
 366                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 367             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 368
 369         stubs.api.collections().create.assert_has_calls([
 370             mock.call(body=JsonDiffMatcher({
 371                 'manifest_text':
 372                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 373                 'replication_desired': None,
 374                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 375             }), ensure_unique_name=False),
 376             mock.call(body=JsonDiffMatcher({
 377                 'manifest_text':
 378                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 379                 'replication_desired': None,
 380                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 381             }), ensure_unique_name=False),
 382             ])
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         stubs.api.container_requests().create.assert_called_with(
 386             body=JsonDiffMatcher(expect_container))
 387         self.assertEqual(stubs.capture_stdout.getvalue(),
 388                          stubs.expect_container_request_uuid + '\n')
 389         self.assertEqual(exited, 0)
 390
 391
 392     @stubs
 393     def test_submit_container_tool(self, stubs):
 394         # test for issue #16139
 395         exited = arvados_cwl.main(
 396             ["--submit", "--no-wait", "--api=containers", "--debug",
 397                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 398             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 399
 400         self.assertEqual(stubs.capture_stdout.getvalue(),
 401                          stubs.expect_container_request_uuid + '\n')
 402         self.assertEqual(exited, 0)
 403
 404     @stubs
 405     def test_submit_container_no_reuse(self, stubs):
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 408                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 410
 411         expect_container = copy.deepcopy(stubs.expect_container_spec)
 412         expect_container["command"] = [
 413             'arvados-cwl-runner', '--local', '--api=containers',
 414             '--no-log-timestamps', '--disable-validate', '--disable-color',
 415             '--eval-timeout=20', '--thread-count=0',
 416             '--disable-reuse', "--collection-cache-size=256",
 417             '--debug', '--on-error=continue',
 418             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 419         expect_container["use_existing"] = False
 420
 421         stubs.api.container_requests().create.assert_called_with(
 422             body=JsonDiffMatcher(expect_container))
 423         self.assertEqual(stubs.capture_stdout.getvalue(),
 424                          stubs.expect_container_request_uuid + '\n')
 425         self.assertEqual(exited, 0)
 426
 427     @stubs
 428     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 429         exited = arvados_cwl.main(
 430             ["--submit", "--no-wait", "--api=containers", "--debug",
 431              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 432             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 433         self.assertEqual(exited, 0)
 434
 435         expect_container = copy.deepcopy(stubs.expect_container_spec)
 436         expect_container["command"] = [
 437             'arvados-cwl-runner', '--local', '--api=containers',
 438             '--no-log-timestamps', '--disable-validate', '--disable-color',
 439             '--eval-timeout=20', '--thread-count=0',
 440             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 441             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 442         expect_container["use_existing"] = False
 443         expect_container["name"] = "submit_wf_no_reuse.cwl"
 444         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 445             {
 446                 "class": "http://arvados.org/cwl#ReuseRequirement",
 447                 "enableReuse": False,
 448             },
 449         ]
 450         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 451             "arv": "http://arvados.org/cwl#",
 452             "cwltool": "http://commonwl.org/cwltool#"
 453         }
 454
 455         stubs.api.container_requests().create.assert_called_with(
 456             body=JsonDiffMatcher(expect_container))
 457         self.assertEqual(stubs.capture_stdout.getvalue(),
 458                          stubs.expect_container_request_uuid + '\n')
 459
 460
 461     @stubs
 462     def test_submit_container_on_error(self, stubs):
 463         exited = arvados_cwl.main(
 464             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 465                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 466             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 467
 468         expect_container = copy.deepcopy(stubs.expect_container_spec)
 469         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 470                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 471                                        '--eval-timeout=20', '--thread-count=0',
 472                                        '--enable-reuse', "--collection-cache-size=256",
 473                                        '--debug', '--on-error=stop',
 474                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 475
 476         stubs.api.container_requests().create.assert_called_with(
 477             body=JsonDiffMatcher(expect_container))
 478         self.assertEqual(stubs.capture_stdout.getvalue(),
 479                          stubs.expect_container_request_uuid + '\n')
 480         self.assertEqual(exited, 0)
 481
 482     @stubs
 483     def test_submit_container_output_name(self, stubs):
 484         output_name = "test_output_name"
 485
 486         exited = arvados_cwl.main(
 487             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 488                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 489             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 490
 491         expect_container = copy.deepcopy(stubs.expect_container_spec)
 492         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 493                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 494                                        '--eval-timeout=20', '--thread-count=0',
 495                                        '--enable-reuse', "--collection-cache-size=256",
 496                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 497                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 498         expect_container["output_name"] = output_name
 499
 500         stubs.api.container_requests().create.assert_called_with(
 501             body=JsonDiffMatcher(expect_container))
 502         self.assertEqual(stubs.capture_stdout.getvalue(),
 503                          stubs.expect_container_request_uuid + '\n')
 504         self.assertEqual(exited, 0)
 505
 506     @stubs
 507     def test_submit_storage_classes(self, stubs):
 508         exited = arvados_cwl.main(
 509             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 510                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 511             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 512
 513         expect_container = copy.deepcopy(stubs.expect_container_spec)
 514         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 515                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 516                                        '--eval-timeout=20', '--thread-count=0',
 517                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 518                                        "--storage-classes=foo", '--on-error=continue',
 519                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 520
 521         stubs.api.container_requests().create.assert_called_with(
 522             body=JsonDiffMatcher(expect_container))
 523         self.assertEqual(stubs.capture_stdout.getvalue(),
 524                          stubs.expect_container_request_uuid + '\n')
 525         self.assertEqual(exited, 0)
 526
 527     @stubs
 528     def test_submit_multiple_storage_classes(self, stubs):
 529         exited = arvados_cwl.main(
 530             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 531                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533
 534         expect_container = copy.deepcopy(stubs.expect_container_spec)
 535         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 536                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 537                                        '--eval-timeout=20', '--thread-count=0',
 538                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 539                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 540                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 541
 542         stubs.api.container_requests().create.assert_called_with(
 543             body=JsonDiffMatcher(expect_container))
 544         self.assertEqual(stubs.capture_stdout.getvalue(),
 545                          stubs.expect_container_request_uuid + '\n')
 546         self.assertEqual(exited, 0)
 547
 548     @mock.patch("cwltool.task_queue.TaskQueue")
 549     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 550     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 551     @stubs
 552     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 553         final_output_c = arvados.collection.Collection()
 554         make_output.return_value = ({},final_output_c)
 555
 556         def set_final_output(job_order, output_callback, runtimeContext):
 557             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 558             return []
 559         job.side_effect = set_final_output
 560
 561         exited = arvados_cwl.main(
 562             ["--debug", "--local", "--storage-classes=foo",
 563                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 564             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 565
 566         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 567         self.assertEqual(exited, 0)
 568
 569     @mock.patch("cwltool.task_queue.TaskQueue")
 570     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 571     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 572     @stubs
 573     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 574         final_output_c = arvados.collection.Collection()
 575         make_output.return_value = ({},final_output_c)
 576         stubs.api.config().get.return_value = {"default": {"Default": True}}
 577
 578         def set_final_output(job_order, output_callback, runtimeContext):
 579             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 580             return []
 581         job.side_effect = set_final_output
 582
 583         exited = arvados_cwl.main(
 584             ["--debug", "--local",
 585                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 586             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 587
 588         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 589         self.assertEqual(exited, 0)
 590
 591     @mock.patch("cwltool.task_queue.TaskQueue")
 592     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 593     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 594     @stubs
 595     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 596         final_output_c = arvados.collection.Collection()
 597         make_output.return_value = ({},final_output_c)
 598
 599         def set_final_output(job_order, output_callback, runtimeContext):
 600             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 601             return []
 602         job.side_effect = set_final_output
 603
 604         exited = arvados_cwl.main(
 605             ["--debug", "--local",
 606                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 607             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 608
 609         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 610         self.assertEqual(exited, 0)
 611
 612     @stubs
 613     def test_submit_container_output_ttl(self, stubs):
 614         exited = arvados_cwl.main(
 615             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 616                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 617             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 618
 619         expect_container = copy.deepcopy(stubs.expect_container_spec)
 620         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 621                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 622                                        '--eval-timeout=20', '--thread-count=0',
 623                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 624                                        '--on-error=continue',
 625                                        "--intermediate-output-ttl=3600",
 626                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(stubs.capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632         self.assertEqual(exited, 0)
 633
 634     @stubs
 635     def test_submit_container_trash_intermediate(self, stubs):
 636         exited = arvados_cwl.main(
 637             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 638                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 639             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 640
 641
 642         expect_container = copy.deepcopy(stubs.expect_container_spec)
 643         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 644                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 645                                        '--eval-timeout=20', '--thread-count=0',
 646                                        '--enable-reuse', "--collection-cache-size=256",
 647                                        '--debug', '--on-error=continue',
 648                                        "--trash-intermediate",
 649                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 650
 651         stubs.api.container_requests().create.assert_called_with(
 652             body=JsonDiffMatcher(expect_container))
 653         self.assertEqual(stubs.capture_stdout.getvalue(),
 654                          stubs.expect_container_request_uuid + '\n')
 655         self.assertEqual(exited, 0)
 656
 657     @stubs
 658     def test_submit_container_output_tags(self, stubs):
 659         output_tags = "tag0,tag1,tag2"
 660
 661         exited = arvados_cwl.main(
 662             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 663                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 665
 666         expect_container = copy.deepcopy(stubs.expect_container_spec)
 667         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 668                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 669                                        '--eval-timeout=20', '--thread-count=0',
 670                                        '--enable-reuse', "--collection-cache-size=256",
 671                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 672                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 673
 674         stubs.api.container_requests().create.assert_called_with(
 675             body=JsonDiffMatcher(expect_container))
 676         self.assertEqual(stubs.capture_stdout.getvalue(),
 677                          stubs.expect_container_request_uuid + '\n')
 678         self.assertEqual(exited, 0)
 679
 680     @stubs
 681     def test_submit_container_runner_ram(self, stubs):
 682         exited = arvados_cwl.main(
 683             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 684                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 685             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 686
 687         expect_container = copy.deepcopy(stubs.expect_container_spec)
 688         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 689
 690         stubs.api.container_requests().create.assert_called_with(
 691             body=JsonDiffMatcher(expect_container))
 692         self.assertEqual(stubs.capture_stdout.getvalue(),
 693                          stubs.expect_container_request_uuid + '\n')
 694         self.assertEqual(exited, 0)
 695
 696     @mock.patch("arvados.collection.CollectionReader")
 697     @mock.patch("time.sleep")
 698     @stubs
 699     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 700         collectionReader().exists.return_value = True
 701         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 702         exited = arvados_cwl.main(
 703             ["--submit", "--no-wait", "--api=containers", "--debug",
 704              "tests/wf/submit_keepref_wf.cwl"],
 705             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 706         self.assertEqual(exited, 0)
 707
 708     @mock.patch("arvados.collection.CollectionReader")
 709     @mock.patch("time.sleep")
 710     @stubs
 711     def test_submit_keepref(self, stubs, tm, reader):
 712         with open("tests/wf/expect_arvworkflow.cwl") as f:
 713             reader().open().__enter__().read.return_value = f.read()
 714
 715         exited = arvados_cwl.main(
 716             ["--submit", "--no-wait", "--api=containers", "--debug",
 717              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 718             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 719
 720         expect_container = {
 721             'priority': 500,
 722             'mounts': {
 723                 '/var/spool/cwl': {
 724                     'writable': True,
 725                     'kind': 'collection'
 726                 },
 727                 'stdout': {
 728                     'path': '/var/spool/cwl/cwl.output.json',
 729                     'kind': 'file'
 730                 },
 731                 '/var/lib/cwl/workflow': {
 732                     'portable_data_hash': '99999999999999999999999999999994+99',
 733                     'kind': 'collection'
 734                 },
 735                 '/var/lib/cwl/cwl.input.json': {
 736                     'content': {
 737                         'x': 'XxX'
 738                     },
 739                     'kind': 'json'
 740                 }
 741             }, 'state': 'Committed',
 742             'output_path': '/var/spool/cwl',
 743             'name': 'expect_arvworkflow.cwl#main',
 744             'container_image': '999999999999999999999999999999d3+99',
 745             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 746                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 747                         '--eval-timeout=20', '--thread-count=0',
 748                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 749                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 750             'cwd': '/var/spool/cwl',
 751             'runtime_constraints': {
 752                 'API': True,
 753                 'vcpus': 1,
 754                 'ram': 1342177280
 755             },
 756             'use_existing': False,
 757             'properties': {},
 758             'secret_mounts': {}
 759         }
 760
 761         stubs.api.container_requests().create.assert_called_with(
 762             body=JsonDiffMatcher(expect_container))
 763         self.assertEqual(stubs.capture_stdout.getvalue(),
 764                          stubs.expect_container_request_uuid + '\n')
 765         self.assertEqual(exited, 0)
 766
 767     @mock.patch("time.sleep")
 768     @stubs
 769     def test_submit_arvworkflow(self, stubs, tm):
 770         with open("tests/wf/expect_arvworkflow.cwl") as f:
 771             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 772
 773         exited = arvados_cwl.main(
 774             ["--submit", "--no-wait", "--api=containers", "--debug",
 775              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 776             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 777
 778         expect_container = {
 779             'priority': 500,
 780             'mounts': {
 781                 '/var/spool/cwl': {
 782                     'writable': True,
 783                     'kind': 'collection'
 784                 },
 785                 'stdout': {
 786                     'path': '/var/spool/cwl/cwl.output.json',
 787                     'kind': 'file'
 788                 },
 789                 '/var/lib/cwl/workflow.json': {
 790                     'kind': 'json',
 791                     'content': {
 792                         'cwlVersion': 'v1.0',
 793                         '$graph': [
 794                             {
 795                                 'id': '#main',
 796                                 'inputs': [
 797                                     {'type': 'string', 'id': '#main/x'}
 798                                 ],
 799                                 'steps': [
 800                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 801                                      'run': '#submit_tool.cwl',
 802                                      'id': '#main/step1',
 803                                      'out': []}
 804                                 ],
 805                                 'class': 'Workflow',
 806                                 'outputs': []
 807                             },
 808                             {
 809                                 'inputs': [
 810                                     {
 811                                         'inputBinding': {'position': 1},
 812                                         'type': 'string',
 813                                         'id': '#submit_tool.cwl/x'}
 814                                 ],
 815                                 'requirements': [
 816                                     {
 817                                         'dockerPull': 'debian:buster-slim',
 818                                         'class': 'DockerRequirement',
 819                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 820                                     }
 821                                 ],
 822                                 'id': '#submit_tool.cwl',
 823                                 'outputs': [],
 824                                 'baseCommand': 'cat',
 825                                 'class': 'CommandLineTool'
 826                             }
 827                         ]
 828                     }
 829                 },
 830                 '/var/lib/cwl/cwl.input.json': {
 831                     'content': {
 832                         'x': 'XxX'
 833                     },
 834                     'kind': 'json'
 835                 }
 836             }, 'state': 'Committed',
 837             'output_path': '/var/spool/cwl',
 838             'name': 'a test workflow',
 839             'container_image': "999999999999999999999999999999d3+99",
 840             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 841                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 842                         '--eval-timeout=20', '--thread-count=0',
 843                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 844                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 845             'cwd': '/var/spool/cwl',
 846             'runtime_constraints': {
 847                 'API': True,
 848                 'vcpus': 1,
 849                 'ram': 1342177280
 850             },
 851             'use_existing': False,
 852             'properties': {
 853                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 854             },
 855             'secret_mounts': {}
 856         }
 857
 858         stubs.api.container_requests().create.assert_called_with(
 859             body=JsonDiffMatcher(expect_container))
 860         self.assertEqual(stubs.capture_stdout.getvalue(),
 861                          stubs.expect_container_request_uuid + '\n')
 862         self.assertEqual(exited, 0)
 863
 864     @stubs
 865     def test_submit_container_name(self, stubs):
 866         exited = arvados_cwl.main(
 867             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 868                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 869             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 870
 871         expect_container = copy.deepcopy(stubs.expect_container_spec)
 872         expect_container["name"] = "hello container 123"
 873
 874         stubs.api.container_requests().create.assert_called_with(
 875             body=JsonDiffMatcher(expect_container))
 876         self.assertEqual(stubs.capture_stdout.getvalue(),
 877                          stubs.expect_container_request_uuid + '\n')
 878         self.assertEqual(exited, 0)
 879
 880     @stubs
 881     def test_submit_missing_input(self, stubs):
 882         exited = arvados_cwl.main(
 883             ["--submit", "--no-wait", "--api=containers", "--debug",
 884              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 885             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 886         self.assertEqual(exited, 0)
 887
 888         exited = arvados_cwl.main(
 889             ["--submit", "--no-wait", "--api=containers", "--debug",
 890              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 891             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 892         self.assertEqual(exited, 1)
 893
 894     @stubs
 895     def test_submit_container_project(self, stubs):
 896         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 897         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 898         exited = arvados_cwl.main(
 899             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 900                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 901             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 902
 903         expect_container = copy.deepcopy(stubs.expect_container_spec)
 904         expect_container["owner_uuid"] = project_uuid
 905         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 906                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 907                                        "--eval-timeout=20", "--thread-count=0",
 908                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 909                                        '--on-error=continue',
 910                                        '--project-uuid='+project_uuid,
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(stubs.capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917         self.assertEqual(exited, 0)
 918
 919     @stubs
 920     def test_submit_container_eval_timeout(self, stubs):
 921         exited = arvados_cwl.main(
 922             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 923                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 924             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 925
 926         expect_container = copy.deepcopy(stubs.expect_container_spec)
 927         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 928                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 929                                        '--eval-timeout=60.0', '--thread-count=0',
 930                                        '--enable-reuse', "--collection-cache-size=256",
 931                                        '--debug', '--on-error=continue',
 932                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 933
 934         stubs.api.container_requests().create.assert_called_with(
 935             body=JsonDiffMatcher(expect_container))
 936         self.assertEqual(stubs.capture_stdout.getvalue(),
 937                          stubs.expect_container_request_uuid + '\n')
 938         self.assertEqual(exited, 0)
 939
 940     @stubs
 941     def test_submit_container_collection_cache(self, stubs):
 942         exited = arvados_cwl.main(
 943             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 944                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 945             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 946
 947         expect_container = copy.deepcopy(stubs.expect_container_spec)
 948         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 949                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 950                                        '--eval-timeout=20', '--thread-count=0',
 951                                        '--enable-reuse', "--collection-cache-size=500",
 952                                        '--debug', '--on-error=continue',
 953                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 954         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 955
 956         stubs.api.container_requests().create.assert_called_with(
 957             body=JsonDiffMatcher(expect_container))
 958         self.assertEqual(stubs.capture_stdout.getvalue(),
 959                          stubs.expect_container_request_uuid + '\n')
 960         self.assertEqual(exited, 0)
 961
 962     @stubs
 963     def test_submit_container_thread_count(self, stubs):
 964         exited = arvados_cwl.main(
 965             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 966                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 967             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 968
 969         expect_container = copy.deepcopy(stubs.expect_container_spec)
 970         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 971                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 972                                        '--eval-timeout=20', '--thread-count=20',
 973                                        '--enable-reuse', "--collection-cache-size=256",
 974                                        '--debug', '--on-error=continue',
 975                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 976
 977         stubs.api.container_requests().create.assert_called_with(
 978             body=JsonDiffMatcher(expect_container))
 979         self.assertEqual(stubs.capture_stdout.getvalue(),
 980                          stubs.expect_container_request_uuid + '\n')
 981         self.assertEqual(exited, 0)
 982
 983     @stubs
 984     def test_submit_container_runner_image(self, stubs):
 985         exited = arvados_cwl.main(
 986             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 987                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 988             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 989
 990         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 991
 992         expect_container = copy.deepcopy(stubs.expect_container_spec)
 993         stubs.api.container_requests().create.assert_called_with(
 994             body=JsonDiffMatcher(expect_container))
 995         self.assertEqual(stubs.capture_stdout.getvalue(),
 996                          stubs.expect_container_request_uuid + '\n')
 997         self.assertEqual(exited, 0)
 998
 999     @stubs
1000     def test_submit_priority(self, stubs):
1001         exited = arvados_cwl.main(
1002             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1003                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1004             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1005
1006         stubs.expect_container_spec["priority"] = 669
1007
1008         expect_container = copy.deepcopy(stubs.expect_container_spec)
1009         stubs.api.container_requests().create.assert_called_with(
1010             body=JsonDiffMatcher(expect_container))
1011         self.assertEqual(stubs.capture_stdout.getvalue(),
1012                          stubs.expect_container_request_uuid + '\n')
1013         self.assertEqual(exited, 0)
1014
1015     @stubs
1016     def test_submit_wf_runner_resources(self, stubs):
1017         exited = arvados_cwl.main(
1018             ["--submit", "--no-wait", "--api=containers", "--debug",
1019                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1020             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1021
1022         expect_container = copy.deepcopy(stubs.expect_container_spec)
1023         expect_container["runtime_constraints"] = {
1024             "API": True,
1025             "vcpus": 2,
1026             "ram": (2000+512) * 2**20
1027         }
1028         expect_container["name"] = "submit_wf_runner_resources.cwl"
1029         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1030             {
1031                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1032                 "coresMin": 2,
1033                 "ramMin": 2000,
1034                 "keep_cache": 512
1035             }
1036         ]
1037         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1038             "arv": "http://arvados.org/cwl#",
1039         }
1040         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1041                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1042                         '--eval-timeout=20', '--thread-count=0',
1043                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1044                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1045
1046         stubs.api.container_requests().create.assert_called_with(
1047             body=JsonDiffMatcher(expect_container))
1048         self.assertEqual(stubs.capture_stdout.getvalue(),
1049                          stubs.expect_container_request_uuid + '\n')
1050         self.assertEqual(exited, 0)
1051
1052     def tearDown(self):
1053         arvados_cwl.arvdocker.arv_docker_clear_cache()
1054
1055     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1056     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1057     @mock.patch("arvados.api")
1058     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1059         arvados_cwl.arvdocker.arv_docker_clear_cache()
1060
1061         arvrunner = mock.MagicMock()
1062         arvrunner.project_uuid = ""
1063         api.return_value = mock.MagicMock()
1064         arvrunner.api = api.return_value
1065         arvrunner.runtimeContext.match_local_docker = False
1066         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1067                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1068                                                                         "link_class": "docker_image_repo+tag",
1069                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1070                                                                         "owner_uuid": "",
1071                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1072                                                             {"items": [{"created_at": "",
1073                                                                         "head_uuid": "",
1074                                                                         "link_class": "docker_image_hash",
1075                                                                         "name": "123456",
1076                                                                         "owner_uuid": "",
1077                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1078         )
1079         find_one_image_hash.return_value = "123456"
1080
1081         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1082                                                                               "owner_uuid": "",
1083                                                                               "manifest_text": "",
1084                                                                               "properties": ""
1085                                                                           }], "items_available": 1, "offset": 0},)
1086         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1087         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1088                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1089         self.assertEqual("9999999999999999999999999999999b+99",
1090                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1091
1092
1093     @stubs
1094     def test_submit_secrets(self, stubs):
1095         exited = arvados_cwl.main(
1096             ["--submit", "--no-wait", "--api=containers", "--debug",
1097                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1098             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1099
1100         expect_container = {
1101             "command": [
1102                 "arvados-cwl-runner",
1103                 "--local",
1104                 "--api=containers",
1105                 "--no-log-timestamps",
1106                 "--disable-validate",
1107                 "--disable-color",
1108                 "--eval-timeout=20",
1109                 '--thread-count=0',
1110                 "--enable-reuse",
1111                 "--collection-cache-size=256",
1112                 '--debug',
1113                 "--on-error=continue",
1114                 "/var/lib/cwl/workflow.json#main",
1115                 "/var/lib/cwl/cwl.input.json"
1116             ],
1117             "container_image": "999999999999999999999999999999d3+99",
1118             "cwd": "/var/spool/cwl",
1119             "mounts": {
1120                 "/var/lib/cwl/cwl.input.json": {
1121                     "content": {
1122                         "pw": {
1123                             "$include": "/secrets/s0"
1124                         }
1125                     },
1126                     "kind": "json"
1127                 },
1128                 "/var/lib/cwl/workflow.json": {
1129                     "content": {
1130                         "$graph": [
1131                             {
1132                                 "arguments": [
1133                                     "md5sum",
1134                                     "example.conf"
1135                                 ],
1136                                 "class": "CommandLineTool",
1137                                 "hints": [
1138                                     {
1139                                         "class": "http://commonwl.org/cwltool#Secrets",
1140                                         "secrets": [
1141                                             "#secret_job.cwl/pw"
1142                                         ]
1143                                     }
1144                                 ],
1145                                 "id": "#secret_job.cwl",
1146                                 "inputs": [
1147                                     {
1148                                         "id": "#secret_job.cwl/pw",
1149                                         "type": "string"
1150                                     }
1151                                 ],
1152                                 "outputs": [
1153                                     {
1154                                         "id": "#secret_job.cwl/out",
1155                                         "type": "File",
1156                                         "outputBinding": {
1157                                               "glob": "hashed_example.txt"
1158                                         }
1159                                     }
1160                                 ],
1161                                 "stdout": "hashed_example.txt",
1162                                 "requirements": [
1163                                     {
1164                                         "class": "InitialWorkDirRequirement",
1165                                         "listing": [
1166                                             {
1167                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1168                                                 "entryname": "example.conf"
1169                                             }
1170                                         ]
1171                                     }
1172                                 ]
1173                             },
1174                             {
1175                                 "class": "Workflow",
1176                                 "hints": [
1177                                     {
1178                                         "class": "DockerRequirement",
1179                                         "dockerPull": "debian:buster-slim",
1180                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1181                                     },
1182                                     {
1183                                         "class": "http://commonwl.org/cwltool#Secrets",
1184                                         "secrets": [
1185                                             "#main/pw"
1186                                         ]
1187                                     }
1188                                 ],
1189                                 "id": "#main",
1190                                 "inputs": [
1191                                     {
1192                                         "id": "#main/pw",
1193                                         "type": "string"
1194                                     }
1195                                 ],
1196                                 "outputs": [
1197                                     {
1198                                         "id": "#main/out",
1199                                         "outputSource": "#main/step1/out",
1200                                         "type": "File"
1201                                     }
1202                                 ],
1203                                 "steps": [
1204                                     {
1205                                         "id": "#main/step1",
1206                                         "in": [
1207                                             {
1208                                                 "id": "#main/step1/pw",
1209                                                 "source": "#main/pw"
1210                                             }
1211                                         ],
1212                                         "out": [
1213                                             "#main/step1/out"
1214                                         ],
1215                                         "run": "#secret_job.cwl"
1216                                     }
1217                                 ]
1218                             }
1219                         ],
1220                         "$namespaces": {
1221                             "cwltool": "http://commonwl.org/cwltool#"
1222                         },
1223                         "cwlVersion": "v1.0"
1224                     },
1225                     "kind": "json"
1226                 },
1227                 "/var/spool/cwl": {
1228                     "kind": "collection",
1229                     "writable": True
1230                 },
1231                 "stdout": {
1232                     "kind": "file",
1233                     "path": "/var/spool/cwl/cwl.output.json"
1234                 }
1235             },
1236             "name": "secret_wf.cwl",
1237             "output_path": "/var/spool/cwl",
1238             "priority": 500,
1239             "properties": {},
1240             "runtime_constraints": {
1241                 "API": True,
1242                 "ram": 1342177280,
1243                 "vcpus": 1
1244             },
1245             "secret_mounts": {
1246                 "/secrets/s0": {
1247                     "content": "blorp",
1248                     "kind": "text"
1249                 }
1250             },
1251             "state": "Committed",
1252             "use_existing": False
1253         }
1254
1255         stubs.api.container_requests().create.assert_called_with(
1256             body=JsonDiffMatcher(expect_container))
1257         self.assertEqual(stubs.capture_stdout.getvalue(),
1258                          stubs.expect_container_request_uuid + '\n')
1259         self.assertEqual(exited, 0)
1260
1261     @stubs
1262     def test_submit_request_uuid(self, stubs):
1263         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1264         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1265
1266         stubs.api.container_requests().update().execute.return_value = {
1267             "uuid": stubs.expect_container_request_uuid,
1268             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1269             "state": "Queued"
1270         }
1271
1272         exited = arvados_cwl.main(
1273             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1274                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1275             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1276
1277         stubs.api.container_requests().update.assert_called_with(
1278             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1279         self.assertEqual(stubs.capture_stdout.getvalue(),
1280                          stubs.expect_container_request_uuid + '\n')
1281         self.assertEqual(exited, 0)
1282
1283     @stubs
1284     def test_submit_container_cluster_id(self, stubs):
1285         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1286
1287         exited = arvados_cwl.main(
1288             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1289                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1290             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1291
1292         expect_container = copy.deepcopy(stubs.expect_container_spec)
1293
1294         stubs.api.container_requests().create.assert_called_with(
1295             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1296         self.assertEqual(stubs.capture_stdout.getvalue(),
1297                          stubs.expect_container_request_uuid + '\n')
1298         self.assertEqual(exited, 0)
1299
1300     @stubs
1301     def test_submit_validate_cluster_id(self, stubs):
1302         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1303         exited = arvados_cwl.main(
1304             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1305              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1306             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1307         self.assertEqual(exited, 1)
1308
1309     @stubs
1310     def test_submit_validate_project_uuid(self, stubs):
1311         # Fails with bad cluster prefix
1312         exited = arvados_cwl.main(
1313             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1315             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1316         self.assertEqual(exited, 1)
1317
1318         # Project lookup fails
1319         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1320         exited = arvados_cwl.main(
1321             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1322              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1323             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1324         self.assertEqual(exited, 1)
1325
1326         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1327         exited = arvados_cwl.main(
1328             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1329              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1330             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1331         self.assertEqual(exited, 0)
1332
1333
1334     @mock.patch("arvados.collection.CollectionReader")
1335     @stubs
1336     def test_submit_uuid_inputs(self, stubs, collectionReader):
1337         collectionReader().exists.return_value = True
1338         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1339         def list_side_effect(**kwargs):
1340             m = mock.MagicMock()
1341             if "count" in kwargs:
1342                 m.execute.return_value = {"items": [
1343                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1344                 ]}
1345             else:
1346                 m.execute.return_value = {"items": []}
1347             return m
1348         stubs.api.collections().list.side_effect = list_side_effect
1349
1350         exited = arvados_cwl.main(
1351             ["--submit", "--no-wait", "--api=containers", "--debug",
1352                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1353             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1354
1355         expect_container = copy.deepcopy(stubs.expect_container_spec)
1356         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1357         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1358         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1359
1360         stubs.api.collections().list.assert_has_calls([
1361             mock.call(count='none',
1362                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1363                       select=['uuid', 'portable_data_hash'])])
1364         stubs.api.container_requests().create.assert_called_with(
1365             body=JsonDiffMatcher(expect_container))
1366         self.assertEqual(stubs.capture_stdout.getvalue(),
1367                          stubs.expect_container_request_uuid + '\n')
1368         self.assertEqual(exited, 0)
1369
1370     @stubs
1371     def test_submit_mismatched_uuid_inputs(self, stubs):
1372         def list_side_effect(**kwargs):
1373             m = mock.MagicMock()
1374             if "count" in kwargs:
1375                 m.execute.return_value = {"items": [
1376                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1377                 ]}
1378             else:
1379                 m.execute.return_value = {"items": []}
1380             return m
1381         stubs.api.collections().list.side_effect = list_side_effect
1382
1383         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1384             capture_stderr = StringIO()
1385             cwltool_logger = logging.getLogger('cwltool')
1386             stderr_logger = logging.StreamHandler(capture_stderr)
1387             cwltool_logger.addHandler(stderr_logger)
1388
1389             try:
1390                 exited = arvados_cwl.main(
1391                     ["--submit", "--no-wait", "--api=containers", "--debug",
1392                         "tests/wf/submit_wf.cwl", infile],
1393                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1394
1395                 self.assertEqual(exited, 1)
1396                 self.assertRegex(
1397                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1398                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1399             finally:
1400                 cwltool_logger.removeHandler(stderr_logger)
1401
1402     @mock.patch("arvados.collection.CollectionReader")
1403     @stubs
1404     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1405         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1406         capture_stderr = StringIO()
1407
1408         cwltool_logger = logging.getLogger('cwltool')
1409         stderr_logger = logging.StreamHandler(capture_stderr)
1410         cwltool_logger.addHandler(stderr_logger)
1411
1412         exited = arvados_cwl.main(
1413             ["--submit", "--no-wait", "--api=containers", "--debug",
1414                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1415             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1416
1417         try:
1418             self.assertEqual(exited, 1)
1419             self.assertRegex(
1420                 capture_stderr.getvalue(),
1421                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1422         finally:
1423             cwltool_logger.removeHandler(stderr_logger)
1424
1425     @stubs
1426     def test_submit_set_process_properties(self, stubs):
1427         exited = arvados_cwl.main(
1428             ["--submit", "--no-wait", "--api=containers", "--debug",
1429                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1430             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1431
1432         expect_container = copy.deepcopy(stubs.expect_container_spec)
1433         expect_container["name"] = "submit_wf_process_properties.cwl"
1434         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1435             {
1436                 "class": "http://arvados.org/cwl#ProcessProperties",
1437                 "processProperties": [
1438                     {"propertyName": "baz",
1439                      "propertyValue": "$(inputs.x.basename)"},
1440                     {"propertyName": "foo",
1441                      "propertyValue": "bar"},
1442                     {"propertyName": "quux",
1443                      "propertyValue": {
1444                          "q1": 1,
1445                          "q2": 2
1446                      }
1447                     }
1448                 ],
1449             }
1450         ]
1451         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1452             "arv": "http://arvados.org/cwl#"
1453         }
1454
1455         expect_container["properties"] = {
1456             "baz": "blorp.txt",
1457             "foo": "bar",
1458             "quux": {
1459                 "q1": 1,
1460                 "q2": 2
1461             }
1462         }
1463
1464         stubs.api.container_requests().create.assert_called_with(
1465             body=JsonDiffMatcher(expect_container))
1466         self.assertEqual(stubs.capture_stdout.getvalue(),
1467                          stubs.expect_container_request_uuid + '\n')
1468         self.assertEqual(exited, 0)
1469
1470
1471     @stubs
1472     def test_submit_enable_preemptible(self, stubs):
1473         exited = arvados_cwl.main(
1474             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1475                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1477
1478         expect_container = copy.deepcopy(stubs.expect_container_spec)
1479         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1480                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1481                         '--eval-timeout=20', '--thread-count=0',
1482                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1483                                        '--enable-preemptible',
1484                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1485
1486         stubs.api.container_requests().create.assert_called_with(
1487             body=JsonDiffMatcher(expect_container))
1488         self.assertEqual(stubs.capture_stdout.getvalue(),
1489                          stubs.expect_container_request_uuid + '\n')
1490         self.assertEqual(exited, 0)
1491
1492     @stubs
1493     def test_submit_disable_preemptible(self, stubs):
1494         exited = arvados_cwl.main(
1495             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1496                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1497             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1498
1499         expect_container = copy.deepcopy(stubs.expect_container_spec)
1500         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1501                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1502                         '--eval-timeout=20', '--thread-count=0',
1503                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1504                                        '--disable-preemptible',
1505                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1506
1507         stubs.api.container_requests().create.assert_called_with(
1508             body=JsonDiffMatcher(expect_container))
1509         self.assertEqual(stubs.capture_stdout.getvalue(),
1510                          stubs.expect_container_request_uuid + '\n')
1511         self.assertEqual(exited, 0)
1512
1513
1514 class TestCreateWorkflow(unittest.TestCase):
1515     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1516     expect_workflow = StripYAMLComments(
1517         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1518
1519     def setUp(self):
1520         cwltool.process._names = set()
1521         arvados_cwl.arvdocker.arv_docker_clear_cache()
1522
1523     @stubs
1524     def test_create(self, stubs):
1525         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1526         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1527
1528         exited = arvados_cwl.main(
1529             ["--create-workflow", "--debug",
1530              "--api=containers",
1531              "--project-uuid", project_uuid,
1532              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1533             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1534
1535         stubs.api.pipeline_templates().create.refute_called()
1536         stubs.api.container_requests().create.refute_called()
1537
1538         body = {
1539             "workflow": {
1540                 "owner_uuid": project_uuid,
1541                 "name": "submit_wf.cwl",
1542                 "description": "",
1543                 "definition": self.expect_workflow,
1544             }
1545         }
1546         stubs.api.workflows().create.assert_called_with(
1547             body=JsonDiffMatcher(body))
1548
1549         self.assertEqual(stubs.capture_stdout.getvalue(),
1550                          stubs.expect_workflow_uuid + '\n')
1551         self.assertEqual(exited, 0)
1552
1553     @stubs
1554     def test_create_name(self, stubs):
1555         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1556         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1557
1558         exited = arvados_cwl.main(
1559             ["--create-workflow", "--debug",
1560              "--api=containers",
1561              "--project-uuid", project_uuid,
1562              "--name", "testing 123",
1563              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1564             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1565
1566         stubs.api.pipeline_templates().create.refute_called()
1567         stubs.api.container_requests().create.refute_called()
1568
1569         body = {
1570             "workflow": {
1571                 "owner_uuid": project_uuid,
1572                 "name": "testing 123",
1573                 "description": "",
1574                 "definition": self.expect_workflow,
1575             }
1576         }
1577         stubs.api.workflows().create.assert_called_with(
1578             body=JsonDiffMatcher(body))
1579
1580         self.assertEqual(stubs.capture_stdout.getvalue(),
1581                          stubs.expect_workflow_uuid + '\n')
1582         self.assertEqual(exited, 0)
1583
1584
1585     @stubs
1586     def test_update(self, stubs):
1587         exited = arvados_cwl.main(
1588             ["--update-workflow", self.existing_workflow_uuid,
1589              "--debug",
1590              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1591             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1592
1593         body = {
1594             "workflow": {
1595                 "name": "submit_wf.cwl",
1596                 "description": "",
1597                 "definition": self.expect_workflow,
1598             }
1599         }
1600         stubs.api.workflows().update.assert_called_with(
1601             uuid=self.existing_workflow_uuid,
1602             body=JsonDiffMatcher(body))
1603         self.assertEqual(stubs.capture_stdout.getvalue(),
1604                          self.existing_workflow_uuid + '\n')
1605         self.assertEqual(exited, 0)
1606
1607     @stubs
1608     def test_update_name(self, stubs):
1609         exited = arvados_cwl.main(
1610             ["--update-workflow", self.existing_workflow_uuid,
1611              "--debug", "--name", "testing 123",
1612              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1613             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1614
1615         body = {
1616             "workflow": {
1617                 "name": "testing 123",
1618                 "description": "",
1619                 "definition": self.expect_workflow,
1620             }
1621         }
1622         stubs.api.workflows().update.assert_called_with(
1623             uuid=self.existing_workflow_uuid,
1624             body=JsonDiffMatcher(body))
1625         self.assertEqual(stubs.capture_stdout.getvalue(),
1626                          self.existing_workflow_uuid + '\n')
1627         self.assertEqual(exited, 0)
1628
1629     @stubs
1630     def test_create_collection_per_tool(self, stubs):
1631         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1632         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1633
1634         exited = arvados_cwl.main(
1635             ["--create-workflow", "--debug",
1636              "--api=containers",
1637              "--project-uuid", project_uuid,
1638              "tests/collection_per_tool/collection_per_tool.cwl"],
1639             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1640
1641         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1642         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1643
1644         body = {
1645             "workflow": {
1646                 "owner_uuid": project_uuid,
1647                 "name": "collection_per_tool.cwl",
1648                 "description": "",
1649                 "definition": expect_workflow,
1650             }
1651         }
1652         stubs.api.workflows().create.assert_called_with(
1653             body=JsonDiffMatcher(body))
1654
1655         self.assertEqual(stubs.capture_stdout.getvalue(),
1656                          stubs.expect_workflow_uuid + '\n')
1657         self.assertEqual(exited, 0)
1658
1659     @stubs
1660     def test_create_with_imports(self, stubs):
1661         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1662         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1663
1664         exited = arvados_cwl.main(
1665             ["--create-workflow", "--debug",
1666              "--api=containers",
1667              "--project-uuid", project_uuid,
1668              "tests/wf/feddemo/feddemo.cwl"],
1669             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1670
1671         stubs.api.pipeline_templates().create.refute_called()
1672         stubs.api.container_requests().create.refute_called()
1673
1674         self.assertEqual(stubs.capture_stdout.getvalue(),
1675                          stubs.expect_workflow_uuid + '\n')
1676         self.assertEqual(exited, 0)
1677
1678     @stubs
1679     def test_create_with_no_input(self, stubs):
1680         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1681         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1682
1683         exited = arvados_cwl.main(
1684             ["--create-workflow", "--debug",
1685              "--api=containers",
1686              "--project-uuid", project_uuid,
1687              "tests/wf/revsort/revsort.cwl"],
1688             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1689
1690         stubs.api.pipeline_templates().create.refute_called()
1691         stubs.api.container_requests().create.refute_called()
1692
1693         self.assertEqual(stubs.capture_stdout.getvalue(),
1694                          stubs.expect_workflow_uuid + '\n')
1695         self.assertEqual(exited, 0)