sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20
  21 from io import BytesIO
  22
  23 # StringIO.StringIO and io.StringIO have different behavior write() is
  24 # called with both python2 (byte) strings and unicode strings
  25 # (specifically there's some logging in cwltool that causes trouble).
  26 # This isn't a problem on python3 because all string are unicode.
  27 if sys.version_info[0] < 3:
  28     from StringIO import StringIO
  29 else:
  30     from io import StringIO
  31
  32 import arvados
  33 import arvados.collection
  34 import arvados_cwl
  35 import arvados_cwl.executor
  36 import arvados_cwl.runner
  37 import arvados.keep
  38
  39 from .matcher import JsonDiffMatcher, StripYAMLComments
  40 from .mock_discovery import get_rootDesc
  41
  42 import ruamel.yaml as yaml
  43
  44 _rootDesc = None
  45
  46 def stubs(func):
  47     @functools.wraps(func)
  48     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  49     @mock.patch("arvados.collection.KeepClient")
  50     @mock.patch("arvados.keep.KeepClient")
  51     @mock.patch("arvados.events.subscribe")
  52     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  53         class Stubs(object):
  54             pass
  55         stubs = Stubs()
  56         stubs.events = events
  57         stubs.keepdocker = keepdocker
  58
  59         def putstub(p, **kwargs):
  60             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  61         keep_client1().put.side_effect = putstub
  62         keep_client1.put.side_effect = putstub
  63         keep_client2().put.side_effect = putstub
  64         keep_client2.put.side_effect = putstub
  65
  66         stubs.keep_client = keep_client2
  67         stubs.docker_images = {
  68             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  69             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  70             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  71             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  72         }
  73         def kd(a, b, image_name=None, image_tag=None):
  74             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  75         stubs.keepdocker.side_effect = kd
  76
  77         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  78         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  79
  80         if sys.version_info[0] < 3:
  81             stubs.capture_stdout = BytesIO()
  82         else:
  83             stubs.capture_stdout = StringIO()
  84
  85         stubs.api = mock.MagicMock()
  86         stubs.api._rootDesc = get_rootDesc()
  87         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  88
  89         stubs.api.users().current().execute.return_value = {
  90             "uuid": stubs.fake_user_uuid,
  91         }
  92         stubs.api.collections().list().execute.return_value = {"items": []}
  93         stubs.api.containers().current().execute.return_value = {
  94             "uuid": stubs.fake_container_uuid,
  95         }
  96
  97         class CollectionExecute(object):
  98             def __init__(self, exe):
  99                 self.exe = exe
 100             def execute(self, num_retries=None):
 101                 return self.exe
 102
 103         def collection_createstub(created_collections, body, ensure_unique_name=None):
 104             mt = body["manifest_text"].encode('utf-8')
 105             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 106             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 107             created_collections[uuid] = {
 108                 "uuid": uuid,
 109                 "portable_data_hash": pdh,
 110                 "manifest_text": mt.decode('utf-8')
 111             }
 112             return CollectionExecute(created_collections[uuid])
 113
 114         def collection_getstub(created_collections, uuid):
 115             for v in viewvalues(created_collections):
 116                 if uuid in (v["uuid"], v["portable_data_hash"]):
 117                     return CollectionExecute(v)
 118
 119         created_collections = {
 120             "99999999999999999999999999999998+99": {
 121                 "uuid": "",
 122                 "portable_data_hash": "99999999999999999999999999999998+99",
 123                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 124             },
 125             "99999999999999999999999999999997+99": {
 126                 "uuid": "",
 127                 "portable_data_hash": "99999999999999999999999999999997+99",
 128                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 129             },
 130             "99999999999999999999999999999994+99": {
 131                 "uuid": "",
 132                 "portable_data_hash": "99999999999999999999999999999994+99",
 133                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 134             },
 135             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 136                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 137                 "portable_data_hash": "999999999999999999999999999999d3+99",
 138                 "manifest_text": ""
 139             },
 140             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 141                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 142                 "portable_data_hash": "999999999999999999999999999999d4+99",
 143                 "manifest_text": ""
 144             },
 145             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 146                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 147                 "portable_data_hash": "999999999999999999999999999999d5+99",
 148                 "manifest_text": ""
 149             },
 150             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 151                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 152                 "portable_data_hash": "999999999999999999999999999999d6+99",
 153                 "manifest_text": ""
 154             }
 155         }
 156         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 157         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 158
 159         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 160         stubs.api.jobs().create().execute.return_value = {
 161             "uuid": stubs.expect_job_uuid,
 162             "state": "Queued",
 163         }
 164
 165         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 166         stubs.api.container_requests().create().execute.return_value = {
 167             "uuid": stubs.expect_container_request_uuid,
 168             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 169             "state": "Queued"
 170         }
 171
 172         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 173         stubs.api.pipeline_templates().create().execute.return_value = {
 174             "uuid": stubs.expect_pipeline_template_uuid,
 175         }
 176         stubs.expect_job_spec = {
 177             'runtime_constraints': {
 178                 'docker_image': '999999999999999999999999999999d3+99',
 179                 'min_ram_mb_per_node': 1024
 180             },
 181             'script_parameters': {
 182                 'x': {
 183                     'basename': 'blorp.txt',
 184                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 185                     'class': 'File'
 186                 },
 187                 'y': {
 188                     'basename': '99999999999999999999999999999998+99',
 189                     'location': 'keep:99999999999999999999999999999998+99',
 190                     'class': 'Directory'
 191                 },
 192                 'z': {
 193                     'basename': 'anonymous',
 194                     "listing": [{
 195                         "basename": "renamed.txt",
 196                         "class": "File",
 197                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 198                         "size": 0
 199                     }],
 200                     'class': 'Directory'
 201                 },
 202                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 203             },
 204             'repository': 'arvados',
 205             'script_version': 'master',
 206             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 207             'script': 'cwl-runner'
 208         }
 209         stubs.pipeline_component = stubs.expect_job_spec.copy()
 210         stubs.expect_pipeline_instance = {
 211             'name': 'submit_wf.cwl',
 212             'state': 'RunningOnServer',
 213             'owner_uuid': None,
 214             "components": {
 215                 "cwl-runner": {
 216                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 217                     'script_parameters': {
 218                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 219                         'x': {"value": {
 220                             'basename': 'blorp.txt',
 221                             'class': 'File',
 222                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 223                             "size": 16
 224                         }},
 225                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 226                               'listing': [
 227                                   {
 228                                       'basename': 'renamed.txt',
 229                                       'class': 'File', 'location':
 230                                       'keep:99999999999999999999999999999998+99/file1.txt',
 231                                       'size': 0
 232                                   }
 233                               ]}},
 234                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 235                         'arv:debug': True,
 236                         'arv:enable_reuse': True,
 237                         'arv:on_error': 'continue'
 238                     },
 239                     'repository': 'arvados',
 240                     'script_version': 'master',
 241                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 242                     'script': 'cwl-runner',
 243                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 244                 }
 245             }
 246         }
 247         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 248         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 249         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 250         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 251         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 252             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 253             "state": "Queued"
 254         }
 255         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 256         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 257
 258         with open("tests/wf/submit_wf_packed.cwl") as f:
 259             expect_packed_workflow = yaml.round_trip_load(f)
 260
 261         stubs.expect_container_spec = {
 262             'priority': 500,
 263             'mounts': {
 264                 '/var/spool/cwl': {
 265                     'writable': True,
 266                     'kind': 'collection'
 267                 },
 268                 '/var/lib/cwl/workflow.json': {
 269                     'content': expect_packed_workflow,
 270                     'kind': 'json'
 271                 },
 272                 'stdout': {
 273                     'path': '/var/spool/cwl/cwl.output.json',
 274                     'kind': 'file'
 275                 },
 276                 '/var/lib/cwl/cwl.input.json': {
 277                     'kind': 'json',
 278                     'content': {
 279                         'y': {
 280                             'basename': '99999999999999999999999999999998+99',
 281                             'location': 'keep:99999999999999999999999999999998+99',
 282                             'class': 'Directory'},
 283                         'x': {
 284                             'basename': u'blorp.txt',
 285                             'class': 'File',
 286                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 287                             "size": 16
 288                         },
 289                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 290                             {'basename': 'renamed.txt',
 291                              'class': 'File',
 292                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 293                              'size': 0
 294                             }
 295                         ]}
 296                     },
 297                     'kind': 'json'
 298                 }
 299             },
 300             'secret_mounts': {},
 301             'state': 'Committed',
 302             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 303                         '--no-log-timestamps', '--disable-validate',
 304                         '--eval-timeout=20', '--thread-count=1',
 305                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 306                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 307             'name': 'submit_wf.cwl',
 308             'container_image': '999999999999999999999999999999d3+99',
 309             'output_path': '/var/spool/cwl',
 310             'cwd': '/var/spool/cwl',
 311             'runtime_constraints': {
 312                 'API': True,
 313                 'vcpus': 1,
 314                 'ram': (1024+256)*1024*1024
 315             },
 316             'use_existing': True,
 317             'properties': {},
 318             'secret_mounts': {}
 319         }
 320
 321         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 322         stubs.api.workflows().create().execute.return_value = {
 323             "uuid": stubs.expect_workflow_uuid,
 324         }
 325         def update_mock(**kwargs):
 326             stubs.updated_uuid = kwargs.get('uuid')
 327             return mock.DEFAULT
 328         stubs.api.workflows().update.side_effect = update_mock
 329         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 330             "uuid": stubs.updated_uuid,
 331         }
 332
 333         return func(self, stubs, *args, **kwargs)
 334     return wrapped
 335
 336
 337 class TestSubmit(unittest.TestCase):
 338     @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
 339     @mock.patch("time.sleep")
 340     @stubs
 341     def test_submit(self, stubs, tm, arvdock):
 342         def get_image(api_client, dockerRequirement, pull_image, project_uuid):
 343             if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
 344                 return '999999999999999999999999999999d3+99'
 345             elif dockerRequirement["dockerPull"] == "debian:8":
 346                 return '999999999999999999999999999999d4+99'
 347         arvdock.side_effect = get_image
 348
 349         exited = arvados_cwl.main(
 350             ["--submit", "--no-wait", "--api=jobs", "--debug",
 351              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 352             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 353
 354         stubs.api.collections().create.assert_has_calls([
 355             mock.call(body=JsonDiffMatcher({
 356                 'manifest_text':
 357                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 358                 'replication_desired': None,
 359                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 360             }), ensure_unique_name=False),
 361             mock.call(body=JsonDiffMatcher({
 362                 'manifest_text':
 363                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 364                 'replication_desired': None,
 365                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 366             }), ensure_unique_name=False),
 367             mock.call(body=JsonDiffMatcher({
 368                 'manifest_text':
 369                 ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
 370                 'replication_desired': None,
 371                 'name': 'submit_wf.cwl',
 372             }), ensure_unique_name=True)        ])
 373
 374         arvdock.assert_has_calls([
 375             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 376             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
 377             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 378         ])
 379
 380         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 381         stubs.api.pipeline_instances().create.assert_called_with(
 382             body=JsonDiffMatcher(expect_pipeline))
 383         self.assertEqual(stubs.capture_stdout.getvalue(),
 384                          stubs.expect_pipeline_uuid + '\n')
 385         self.assertEqual(exited, 0)
 386
 387     @mock.patch("time.sleep")
 388     @stubs
 389     def test_submit_no_reuse(self, stubs, tm):
 390         exited = arvados_cwl.main(
 391             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 392              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 393             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 394
 395         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 396         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 397         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 398
 399         stubs.api.pipeline_instances().create.assert_called_with(
 400             body=JsonDiffMatcher(expect_pipeline))
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_pipeline_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_error_when_multiple_storage_classes_specified(self, stubs):
 407         storage_classes = "foo,bar"
 408         exited = arvados_cwl.main(
 409                 ["--debug", "--storage-classes", storage_classes,
 410                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411                 sys.stdin, sys.stderr, api_client=stubs.api)
 412         self.assertEqual(exited, 1)
 413
 414     @mock.patch("time.sleep")
 415     @stubs
 416     def test_submit_on_error(self, stubs, tm):
 417         exited = arvados_cwl.main(
 418             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 419              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 420             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 421
 422         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 423         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 424
 425         stubs.api.pipeline_instances().create.assert_called_with(
 426             body=JsonDiffMatcher(expect_pipeline))
 427         self.assertEqual(stubs.capture_stdout.getvalue(),
 428                          stubs.expect_pipeline_uuid + '\n')
 429         self.assertEqual(exited, 0)
 430
 431     @mock.patch("time.sleep")
 432     @stubs
 433     def test_submit_runner_ram(self, stubs, tm):
 434         exited = arvados_cwl.main(
 435             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 436              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 437             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 438
 439         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 440         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 441
 442         stubs.api.pipeline_instances().create.assert_called_with(
 443             body=JsonDiffMatcher(expect_pipeline))
 444         self.assertEqual(stubs.capture_stdout.getvalue(),
 445                          stubs.expect_pipeline_uuid + '\n')
 446         self.assertEqual(exited, 0)
 447
 448     @mock.patch("time.sleep")
 449     @stubs
 450     def test_submit_invalid_runner_ram(self, stubs, tm):
 451         exited = arvados_cwl.main(
 452             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 453              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 454             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 455         self.assertEqual(exited, 1)
 456
 457     @mock.patch("time.sleep")
 458     @stubs
 459     def test_submit_output_name(self, stubs, tm):
 460         output_name = "test_output_name"
 461
 462         exited = arvados_cwl.main(
 463             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 464              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 465             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 466
 467         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 468         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 469
 470         stubs.api.pipeline_instances().create.assert_called_with(
 471             body=JsonDiffMatcher(expect_pipeline))
 472         self.assertEqual(stubs.capture_stdout.getvalue(),
 473                          stubs.expect_pipeline_uuid + '\n')
 474         self.assertEqual(exited, 0)
 475
 476     @mock.patch("time.sleep")
 477     @stubs
 478     def test_submit_pipeline_name(self, stubs, tm):
 479         exited = arvados_cwl.main(
 480             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 481              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 482             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 483         self.assertEqual(exited, 0)
 484
 485         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 486         expect_pipeline["name"] = "hello job 123"
 487
 488         stubs.api.pipeline_instances().create.assert_called_with(
 489             body=JsonDiffMatcher(expect_pipeline))
 490         self.assertEqual(stubs.capture_stdout.getvalue(),
 491                          stubs.expect_pipeline_uuid + '\n')
 492
 493     @mock.patch("time.sleep")
 494     @stubs
 495     def test_submit_output_tags(self, stubs, tm):
 496         output_tags = "tag0,tag1,tag2"
 497
 498         exited = arvados_cwl.main(
 499             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 500              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 501             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 502         self.assertEqual(exited, 0)
 503
 504         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 505         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 506
 507         stubs.api.pipeline_instances().create.assert_called_with(
 508             body=JsonDiffMatcher(expect_pipeline))
 509         self.assertEqual(stubs.capture_stdout.getvalue(),
 510                          stubs.expect_pipeline_uuid + '\n')
 511
 512     @mock.patch("time.sleep")
 513     @stubs
 514     def test_submit_with_project_uuid(self, stubs, tm):
 515         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 516
 517         exited = arvados_cwl.main(
 518             ["--submit", "--no-wait", "--debug",
 519              "--project-uuid", project_uuid,
 520              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 521             sys.stdout, sys.stderr, api_client=stubs.api)
 522         self.assertEqual(exited, 0)
 523
 524         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 525         expect_pipeline["owner_uuid"] = project_uuid
 526         stubs.api.pipeline_instances().create.assert_called_with(
 527             body=JsonDiffMatcher(expect_pipeline))
 528
 529     @stubs
 530     def test_submit_container(self, stubs):
 531         exited = arvados_cwl.main(
 532             ["--submit", "--no-wait", "--api=containers", "--debug",
 533                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 534             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 535
 536         stubs.api.collections().create.assert_has_calls([
 537             mock.call(body=JsonDiffMatcher({
 538                 'manifest_text':
 539                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 540                 'replication_desired': None,
 541                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 542             }), ensure_unique_name=False),
 543             mock.call(body=JsonDiffMatcher({
 544                 'manifest_text':
 545                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 546                 'replication_desired': None,
 547                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 548             }), ensure_unique_name=False)])
 549
 550         expect_container = copy.deepcopy(stubs.expect_container_spec)
 551         stubs.api.container_requests().create.assert_called_with(
 552             body=JsonDiffMatcher(expect_container))
 553         self.assertEqual(stubs.capture_stdout.getvalue(),
 554                          stubs.expect_container_request_uuid + '\n')
 555         self.assertEqual(exited, 0)
 556
 557     @stubs
 558     def test_submit_container_no_reuse(self, stubs):
 559         exited = arvados_cwl.main(
 560             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 561                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 562             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 563
 564         expect_container = copy.deepcopy(stubs.expect_container_spec)
 565         expect_container["command"] = [
 566             'arvados-cwl-runner', '--local', '--api=containers',
 567             '--no-log-timestamps', '--disable-validate',
 568             '--eval-timeout=20', '--thread-count=1',
 569             '--disable-reuse', "--collection-cache-size=256",
 570             '--debug', '--on-error=continue',
 571             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 572         expect_container["use_existing"] = False
 573
 574         stubs.api.container_requests().create.assert_called_with(
 575             body=JsonDiffMatcher(expect_container))
 576         self.assertEqual(stubs.capture_stdout.getvalue(),
 577                          stubs.expect_container_request_uuid + '\n')
 578         self.assertEqual(exited, 0)
 579
 580     @stubs
 581     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 582         exited = arvados_cwl.main(
 583             ["--submit", "--no-wait", "--api=containers", "--debug",
 584              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 585             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 586         self.assertEqual(exited, 0)
 587
 588         expect_container = copy.deepcopy(stubs.expect_container_spec)
 589         expect_container["command"] = [
 590             'arvados-cwl-runner', '--local', '--api=containers',
 591             '--no-log-timestamps', '--disable-validate',
 592             '--eval-timeout=20', '--thread-count=1',
 593             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 594             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 595         expect_container["use_existing"] = False
 596         expect_container["name"] = "submit_wf_no_reuse.cwl"
 597         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 598             {
 599                 "class": "http://arvados.org/cwl#ReuseRequirement",
 600                 "enableReuse": False,
 601             },
 602         ]
 603         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 604             "arv": "http://arvados.org/cwl#",
 605             "cwltool": "http://commonwl.org/cwltool#"
 606         }
 607
 608         stubs.api.container_requests().create.assert_called_with(
 609             body=JsonDiffMatcher(expect_container))
 610         self.assertEqual(stubs.capture_stdout.getvalue(),
 611                          stubs.expect_container_request_uuid + '\n')
 612
 613
 614     @stubs
 615     def test_submit_container_on_error(self, stubs):
 616         exited = arvados_cwl.main(
 617             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 618                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 619             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 620
 621         expect_container = copy.deepcopy(stubs.expect_container_spec)
 622         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 623                                        '--no-log-timestamps', '--disable-validate',
 624                                        '--eval-timeout=20', '--thread-count=1',
 625                                        '--enable-reuse', "--collection-cache-size=256",
 626                                        '--debug', '--on-error=stop',
 627                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 628
 629         stubs.api.container_requests().create.assert_called_with(
 630             body=JsonDiffMatcher(expect_container))
 631         self.assertEqual(stubs.capture_stdout.getvalue(),
 632                          stubs.expect_container_request_uuid + '\n')
 633         self.assertEqual(exited, 0)
 634
 635     @stubs
 636     def test_submit_container_output_name(self, stubs):
 637         output_name = "test_output_name"
 638
 639         exited = arvados_cwl.main(
 640             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 641                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 642             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 643
 644         expect_container = copy.deepcopy(stubs.expect_container_spec)
 645         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 646                                        '--no-log-timestamps', '--disable-validate',
 647                                        '--eval-timeout=20', '--thread-count=1',
 648                                        '--enable-reuse', "--collection-cache-size=256",
 649                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 650                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 651         expect_container["output_name"] = output_name
 652
 653         stubs.api.container_requests().create.assert_called_with(
 654             body=JsonDiffMatcher(expect_container))
 655         self.assertEqual(stubs.capture_stdout.getvalue(),
 656                          stubs.expect_container_request_uuid + '\n')
 657         self.assertEqual(exited, 0)
 658
 659     @stubs
 660     def test_submit_storage_classes(self, stubs):
 661         exited = arvados_cwl.main(
 662             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 663                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 664             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 665
 666         expect_container = copy.deepcopy(stubs.expect_container_spec)
 667         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 668                                        '--no-log-timestamps', '--disable-validate',
 669                                        '--eval-timeout=20', '--thread-count=1',
 670                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 671                                        "--storage-classes=foo", '--on-error=continue',
 672                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 673
 674         stubs.api.container_requests().create.assert_called_with(
 675             body=JsonDiffMatcher(expect_container))
 676         self.assertEqual(stubs.capture_stdout.getvalue(),
 677                          stubs.expect_container_request_uuid + '\n')
 678         self.assertEqual(exited, 0)
 679
 680     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 681     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 682     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 683     @stubs
 684     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 685         def set_final_output(job_order, output_callback, runtimeContext):
 686             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 687             return []
 688         job.side_effect = set_final_output
 689
 690         exited = arvados_cwl.main(
 691             ["--debug", "--local", "--storage-classes=foo",
 692                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 693             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 694
 695         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 696         self.assertEqual(exited, 0)
 697
 698     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 699     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 700     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 701     @stubs
 702     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 703         def set_final_output(job_order, output_callback, runtimeContext):
 704             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 705             return []
 706         job.side_effect = set_final_output
 707
 708         exited = arvados_cwl.main(
 709             ["--debug", "--local",
 710                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 711             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 712
 713         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 714         self.assertEqual(exited, 0)
 715
 716     @stubs
 717     def test_submit_container_output_ttl(self, stubs):
 718         exited = arvados_cwl.main(
 719             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 720                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 721             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 722
 723         expect_container = copy.deepcopy(stubs.expect_container_spec)
 724         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 725                                        '--no-log-timestamps', '--disable-validate',
 726                                        '--eval-timeout=20', '--thread-count=1',
 727                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 728                                        '--on-error=continue',
 729                                        "--intermediate-output-ttl=3600",
 730                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 731
 732         stubs.api.container_requests().create.assert_called_with(
 733             body=JsonDiffMatcher(expect_container))
 734         self.assertEqual(stubs.capture_stdout.getvalue(),
 735                          stubs.expect_container_request_uuid + '\n')
 736         self.assertEqual(exited, 0)
 737
 738     @stubs
 739     def test_submit_container_trash_intermediate(self, stubs):
 740         exited = arvados_cwl.main(
 741             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 742                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 743             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 744
 745
 746         expect_container = copy.deepcopy(stubs.expect_container_spec)
 747         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 748                                        '--no-log-timestamps', '--disable-validate',
 749                                        '--eval-timeout=20', '--thread-count=1',
 750                                        '--enable-reuse', "--collection-cache-size=256",
 751                                        '--debug', '--on-error=continue',
 752                                        "--trash-intermediate",
 753                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 754
 755         stubs.api.container_requests().create.assert_called_with(
 756             body=JsonDiffMatcher(expect_container))
 757         self.assertEqual(stubs.capture_stdout.getvalue(),
 758                          stubs.expect_container_request_uuid + '\n')
 759         self.assertEqual(exited, 0)
 760
 761     @stubs
 762     def test_submit_container_output_tags(self, stubs):
 763         output_tags = "tag0,tag1,tag2"
 764
 765         exited = arvados_cwl.main(
 766             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 767                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 768             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 769
 770         expect_container = copy.deepcopy(stubs.expect_container_spec)
 771         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 772                                        '--no-log-timestamps', '--disable-validate',
 773                                        '--eval-timeout=20', '--thread-count=1',
 774                                        '--enable-reuse', "--collection-cache-size=256",
 775                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 776                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 777
 778         stubs.api.container_requests().create.assert_called_with(
 779             body=JsonDiffMatcher(expect_container))
 780         self.assertEqual(stubs.capture_stdout.getvalue(),
 781                          stubs.expect_container_request_uuid + '\n')
 782         self.assertEqual(exited, 0)
 783
 784     @stubs
 785     def test_submit_container_runner_ram(self, stubs):
 786         exited = arvados_cwl.main(
 787             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 788                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 789             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 790
 791         expect_container = copy.deepcopy(stubs.expect_container_spec)
 792         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 793
 794         stubs.api.container_requests().create.assert_called_with(
 795             body=JsonDiffMatcher(expect_container))
 796         self.assertEqual(stubs.capture_stdout.getvalue(),
 797                          stubs.expect_container_request_uuid + '\n')
 798         self.assertEqual(exited, 0)
 799
 800     @mock.patch("arvados.collection.CollectionReader")
 801     @mock.patch("time.sleep")
 802     @stubs
 803     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 804         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 805         exited = arvados_cwl.main(
 806             ["--submit", "--no-wait", "--api=containers", "--debug",
 807              "tests/wf/submit_keepref_wf.cwl"],
 808             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 809         self.assertEqual(exited, 0)
 810
 811     @mock.patch("arvados.collection.CollectionReader")
 812     @mock.patch("time.sleep")
 813     @stubs
 814     def test_submit_keepref(self, stubs, tm, reader):
 815         with open("tests/wf/expect_arvworkflow.cwl") as f:
 816             reader().open().__enter__().read.return_value = f.read()
 817
 818         exited = arvados_cwl.main(
 819             ["--submit", "--no-wait", "--api=containers", "--debug",
 820              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 821             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 822
 823         expect_container = {
 824             'priority': 500,
 825             'mounts': {
 826                 '/var/spool/cwl': {
 827                     'writable': True,
 828                     'kind': 'collection'
 829                 },
 830                 'stdout': {
 831                     'path': '/var/spool/cwl/cwl.output.json',
 832                     'kind': 'file'
 833                 },
 834                 '/var/lib/cwl/workflow': {
 835                     'portable_data_hash': '99999999999999999999999999999994+99',
 836                     'kind': 'collection'
 837                 },
 838                 '/var/lib/cwl/cwl.input.json': {
 839                     'content': {
 840                         'x': 'XxX'
 841                     },
 842                     'kind': 'json'
 843                 }
 844             }, 'state': 'Committed',
 845             'output_path': '/var/spool/cwl',
 846             'name': 'expect_arvworkflow.cwl#main',
 847             'container_image': '999999999999999999999999999999d3+99',
 848             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 849                         '--no-log-timestamps', '--disable-validate',
 850                         '--eval-timeout=20', '--thread-count=1',
 851                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 852                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 853             'cwd': '/var/spool/cwl',
 854             'runtime_constraints': {
 855                 'API': True,
 856                 'vcpus': 1,
 857                 'ram': 1342177280
 858             },
 859             'use_existing': True,
 860             'properties': {},
 861             'secret_mounts': {}
 862         }
 863
 864         stubs.api.container_requests().create.assert_called_with(
 865             body=JsonDiffMatcher(expect_container))
 866         self.assertEqual(stubs.capture_stdout.getvalue(),
 867                          stubs.expect_container_request_uuid + '\n')
 868         self.assertEqual(exited, 0)
 869
 870     @mock.patch("arvados.collection.CollectionReader")
 871     @mock.patch("time.sleep")
 872     @stubs
 873     def test_submit_jobs_keepref(self, stubs, tm, reader):
 874         with open("tests/wf/expect_arvworkflow.cwl") as f:
 875             reader().open().__enter__().read.return_value = f.read()
 876
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=jobs", "--debug",
 879              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 881
 882         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 883         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 884         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 885         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 886         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 887         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 888         stubs.api.pipeline_instances().create.assert_called_with(
 889             body=JsonDiffMatcher(expect_pipeline))
 890         self.assertEqual(exited, 0)
 891
 892     @mock.patch("time.sleep")
 893     @stubs
 894     def test_submit_arvworkflow(self, stubs, tm):
 895         with open("tests/wf/expect_arvworkflow.cwl") as f:
 896             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 897
 898         exited = arvados_cwl.main(
 899             ["--submit", "--no-wait", "--api=containers", "--debug",
 900              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 901             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 902
 903         expect_container = {
 904             'priority': 500,
 905             'mounts': {
 906                 '/var/spool/cwl': {
 907                     'writable': True,
 908                     'kind': 'collection'
 909                 },
 910                 'stdout': {
 911                     'path': '/var/spool/cwl/cwl.output.json',
 912                     'kind': 'file'
 913                 },
 914                 '/var/lib/cwl/workflow.json': {
 915                     'kind': 'json',
 916                     'content': {
 917                         'cwlVersion': 'v1.0',
 918                         '$graph': [
 919                             {
 920                                 'id': '#main',
 921                                 'inputs': [
 922                                     {'type': 'string', 'id': '#main/x'}
 923                                 ],
 924                                 'steps': [
 925                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 926                                      'run': '#submit_tool.cwl',
 927                                      'id': '#main/step1',
 928                                      'out': []}
 929                                 ],
 930                                 'class': 'Workflow',
 931                                 'outputs': []
 932                             },
 933                             {
 934                                 'inputs': [
 935                                     {
 936                                         'inputBinding': {'position': 1},
 937                                         'type': 'string',
 938                                         'id': '#submit_tool.cwl/x'}
 939                                 ],
 940                                 'requirements': [
 941                                     {
 942                                         'dockerPull': 'debian:8',
 943                                         'class': 'DockerRequirement',
 944                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 945                                     }
 946                                 ],
 947                                 'id': '#submit_tool.cwl',
 948                                 'outputs': [],
 949                                 'baseCommand': 'cat',
 950                                 'class': 'CommandLineTool'
 951                             }
 952                         ]
 953                     }
 954                 },
 955                 '/var/lib/cwl/cwl.input.json': {
 956                     'content': {
 957                         'x': 'XxX'
 958                     },
 959                     'kind': 'json'
 960                 }
 961             }, 'state': 'Committed',
 962             'output_path': '/var/spool/cwl',
 963             'name': 'a test workflow',
 964             'container_image': "999999999999999999999999999999d3+99",
 965             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 966                         '--no-log-timestamps', '--disable-validate',
 967                         '--eval-timeout=20', '--thread-count=1',
 968                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 969                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 970             'cwd': '/var/spool/cwl',
 971             'runtime_constraints': {
 972                 'API': True,
 973                 'vcpus': 1,
 974                 'ram': 1342177280
 975             },
 976             'use_existing': True,
 977             'properties': {
 978                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 979             },
 980             'secret_mounts': {}
 981         }
 982
 983         stubs.api.container_requests().create.assert_called_with(
 984             body=JsonDiffMatcher(expect_container))
 985         self.assertEqual(stubs.capture_stdout.getvalue(),
 986                          stubs.expect_container_request_uuid + '\n')
 987         self.assertEqual(exited, 0)
 988
 989     @stubs
 990     def test_submit_container_name(self, stubs):
 991         exited = arvados_cwl.main(
 992             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 993                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 995
 996         expect_container = copy.deepcopy(stubs.expect_container_spec)
 997         expect_container["name"] = "hello container 123"
 998
 999         stubs.api.container_requests().create.assert_called_with(
1000             body=JsonDiffMatcher(expect_container))
1001         self.assertEqual(stubs.capture_stdout.getvalue(),
1002                          stubs.expect_container_request_uuid + '\n')
1003         self.assertEqual(exited, 0)
1004
1005     @stubs
1006     def test_submit_missing_input(self, stubs):
1007         exited = arvados_cwl.main(
1008             ["--submit", "--no-wait", "--api=containers", "--debug",
1009              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1010             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1011         self.assertEqual(exited, 0)
1012
1013         exited = arvados_cwl.main(
1014             ["--submit", "--no-wait", "--api=containers", "--debug",
1015              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
1016             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1017         self.assertEqual(exited, 1)
1018
1019     @stubs
1020     def test_submit_container_project(self, stubs):
1021         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1022         exited = arvados_cwl.main(
1023             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1024                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1025             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1026
1027         expect_container = copy.deepcopy(stubs.expect_container_spec)
1028         expect_container["owner_uuid"] = project_uuid
1029         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1030                                        '--no-log-timestamps', '--disable-validate',
1031                                        "--eval-timeout=20", "--thread-count=1",
1032                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
1033                                        '--on-error=continue',
1034                                        '--project-uuid='+project_uuid,
1035                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1036
1037         stubs.api.container_requests().create.assert_called_with(
1038             body=JsonDiffMatcher(expect_container))
1039         self.assertEqual(stubs.capture_stdout.getvalue(),
1040                          stubs.expect_container_request_uuid + '\n')
1041         self.assertEqual(exited, 0)
1042
1043     @stubs
1044     def test_submit_container_eval_timeout(self, stubs):
1045         exited = arvados_cwl.main(
1046             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1047                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1048             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1049
1050         expect_container = copy.deepcopy(stubs.expect_container_spec)
1051         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1052                                        '--no-log-timestamps', '--disable-validate',
1053                                        '--eval-timeout=60.0', '--thread-count=1',
1054                                        '--enable-reuse', "--collection-cache-size=256",
1055                                        '--debug', '--on-error=continue',
1056                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1057
1058         stubs.api.container_requests().create.assert_called_with(
1059             body=JsonDiffMatcher(expect_container))
1060         self.assertEqual(stubs.capture_stdout.getvalue(),
1061                          stubs.expect_container_request_uuid + '\n')
1062         self.assertEqual(exited, 0)
1063
1064     @stubs
1065     def test_submit_container_collection_cache(self, stubs):
1066         exited = arvados_cwl.main(
1067             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
1068                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1069             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1070
1071         expect_container = copy.deepcopy(stubs.expect_container_spec)
1072         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1073                                        '--no-log-timestamps', '--disable-validate',
1074                                        '--eval-timeout=20', '--thread-count=1',
1075                                        '--enable-reuse', "--collection-cache-size=500",
1076                                        '--debug', '--on-error=continue',
1077                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1078         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1079
1080         stubs.api.container_requests().create.assert_called_with(
1081             body=JsonDiffMatcher(expect_container))
1082         self.assertEqual(stubs.capture_stdout.getvalue(),
1083                          stubs.expect_container_request_uuid + '\n')
1084         self.assertEqual(exited, 0)
1085
1086     @stubs
1087     def test_submit_container_thread_count(self, stubs):
1088         exited = arvados_cwl.main(
1089             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1090                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1091             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1092
1093         expect_container = copy.deepcopy(stubs.expect_container_spec)
1094         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1095                                        '--no-log-timestamps', '--disable-validate',
1096                                        '--eval-timeout=20', '--thread-count=20',
1097                                        '--enable-reuse', "--collection-cache-size=256",
1098                                        '--debug', '--on-error=continue',
1099                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1100
1101         stubs.api.container_requests().create.assert_called_with(
1102             body=JsonDiffMatcher(expect_container))
1103         self.assertEqual(stubs.capture_stdout.getvalue(),
1104                          stubs.expect_container_request_uuid + '\n')
1105         self.assertEqual(exited, 0)
1106
1107     @stubs
1108     def test_submit_job_runner_image(self, stubs):
1109         exited = arvados_cwl.main(
1110             ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1111                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1112             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1113
1114         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
1115
1116         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1117         stubs.api.pipeline_instances().create.assert_called_with(
1118             body=JsonDiffMatcher(expect_pipeline))
1119         self.assertEqual(stubs.capture_stdout.getvalue(),
1120                          stubs.expect_pipeline_uuid + '\n')
1121         self.assertEqual(exited, 0)
1122
1123     @stubs
1124     def test_submit_container_runner_image(self, stubs):
1125         exited = arvados_cwl.main(
1126             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1127                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1128             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1129
1130         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1131
1132         expect_container = copy.deepcopy(stubs.expect_container_spec)
1133         stubs.api.container_requests().create.assert_called_with(
1134             body=JsonDiffMatcher(expect_container))
1135         self.assertEqual(stubs.capture_stdout.getvalue(),
1136                          stubs.expect_container_request_uuid + '\n')
1137         self.assertEqual(exited, 0)
1138
1139     @stubs
1140     def test_submit_priority(self, stubs):
1141         exited = arvados_cwl.main(
1142             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1143                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1144             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1145
1146         stubs.expect_container_spec["priority"] = 669
1147
1148         expect_container = copy.deepcopy(stubs.expect_container_spec)
1149         stubs.api.container_requests().create.assert_called_with(
1150             body=JsonDiffMatcher(expect_container))
1151         self.assertEqual(stubs.capture_stdout.getvalue(),
1152                          stubs.expect_container_request_uuid + '\n')
1153         self.assertEqual(exited, 0)
1154
1155     @stubs
1156     def test_submit_wf_runner_resources(self, stubs):
1157         exited = arvados_cwl.main(
1158             ["--submit", "--no-wait", "--api=containers", "--debug",
1159                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1160             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1161
1162         expect_container = copy.deepcopy(stubs.expect_container_spec)
1163         expect_container["runtime_constraints"] = {
1164             "API": True,
1165             "vcpus": 2,
1166             "ram": (2000+512) * 2**20
1167         }
1168         expect_container["name"] = "submit_wf_runner_resources.cwl"
1169         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1170             {
1171                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1172                 "coresMin": 2,
1173                 "ramMin": 2000,
1174                 "keep_cache": 512
1175             }
1176         ]
1177         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1178             "arv": "http://arvados.org/cwl#",
1179         }
1180         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1181                         '--no-log-timestamps', '--disable-validate',
1182                         '--eval-timeout=20', '--thread-count=1',
1183                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1184                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1185
1186         stubs.api.container_requests().create.assert_called_with(
1187             body=JsonDiffMatcher(expect_container))
1188         self.assertEqual(stubs.capture_stdout.getvalue(),
1189                          stubs.expect_container_request_uuid + '\n')
1190         self.assertEqual(exited, 0)
1191
1192     def tearDown(self):
1193         arvados_cwl.arvdocker.arv_docker_clear_cache()
1194
1195     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1196     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1197     @mock.patch("arvados.api")
1198     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1199         arvados_cwl.arvdocker.arv_docker_clear_cache()
1200
1201         arvrunner = mock.MagicMock()
1202         arvrunner.project_uuid = ""
1203         api.return_value = mock.MagicMock()
1204         arvrunner.api = api.return_value
1205         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1206                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1207                                                                         "link_class": "docker_image_repo+tag",
1208                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1209                                                                         "owner_uuid": "",
1210                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1211                                                             {"items": [{"created_at": "",
1212                                                                         "head_uuid": "",
1213                                                                         "link_class": "docker_image_hash",
1214                                                                         "name": "123456",
1215                                                                         "owner_uuid": "",
1216                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1217         )
1218         find_one_image_hash.return_value = "123456"
1219
1220         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1221                                                                               "owner_uuid": "",
1222                                                                               "manifest_text": "",
1223                                                                               "properties": ""
1224                                                                           }], "items_available": 1, "offset": 0},)
1225         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1226         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1227                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1228         self.assertEqual("9999999999999999999999999999999b+99",
1229                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1230
1231
1232     @stubs
1233     def test_submit_secrets(self, stubs):
1234         exited = arvados_cwl.main(
1235             ["--submit", "--no-wait", "--api=containers", "--debug",
1236                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1237             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1238
1239         expect_container = {
1240             "command": [
1241                 "arvados-cwl-runner",
1242                 "--local",
1243                 "--api=containers",
1244                 "--no-log-timestamps",
1245                 "--disable-validate",
1246                 "--eval-timeout=20",
1247                 '--thread-count=1',
1248                 "--enable-reuse",
1249                 "--collection-cache-size=256",
1250                 '--debug',
1251                 "--on-error=continue",
1252                 "/var/lib/cwl/workflow.json#main",
1253                 "/var/lib/cwl/cwl.input.json"
1254             ],
1255             "container_image": "999999999999999999999999999999d3+99",
1256             "cwd": "/var/spool/cwl",
1257             "mounts": {
1258                 "/var/lib/cwl/cwl.input.json": {
1259                     "content": {
1260                         "pw": {
1261                             "$include": "/secrets/s0"
1262                         }
1263                     },
1264                     "kind": "json"
1265                 },
1266                 "/var/lib/cwl/workflow.json": {
1267                     "content": {
1268                         "$graph": [
1269                             {
1270                                 "$namespaces": {
1271                                     "cwltool": "http://commonwl.org/cwltool#"
1272                                 },
1273                                 "arguments": [
1274                                     "md5sum",
1275                                     "example.conf"
1276                                 ],
1277                                 "class": "CommandLineTool",
1278                                 "hints": [
1279                                     {
1280                                         "class": "http://commonwl.org/cwltool#Secrets",
1281                                         "secrets": [
1282                                             "#secret_job.cwl/pw"
1283                                         ]
1284                                     }
1285                                 ],
1286                                 "id": "#secret_job.cwl",
1287                                 "inputs": [
1288                                     {
1289                                         "id": "#secret_job.cwl/pw",
1290                                         "type": "string"
1291                                     }
1292                                 ],
1293                                 "outputs": [
1294                                     {
1295                                         "id": "#secret_job.cwl/out",
1296                                         "type": "stdout"
1297                                     }
1298                                 ],
1299                                 "stdout": "hashed_example.txt",
1300                                 "requirements": [
1301                                     {
1302                                         "class": "InitialWorkDirRequirement",
1303                                         "listing": [
1304                                             {
1305                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1306                                                 "entryname": "example.conf"
1307                                             }
1308                                         ]
1309                                     }
1310                                 ]
1311                             },
1312                             {
1313                                 "class": "Workflow",
1314                                 "hints": [
1315                                     {
1316                                         "class": "DockerRequirement",
1317                                         "dockerPull": "debian:8",
1318                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1319                                     },
1320                                     {
1321                                         "class": "http://commonwl.org/cwltool#Secrets",
1322                                         "secrets": [
1323                                             "#main/pw"
1324                                         ]
1325                                     }
1326                                 ],
1327                                 "id": "#main",
1328                                 "inputs": [
1329                                     {
1330                                         "id": "#main/pw",
1331                                         "type": "string"
1332                                     }
1333                                 ],
1334                                 "outputs": [
1335                                     {
1336                                         "id": "#main/out",
1337                                         "outputSource": "#main/step1/out",
1338                                         "type": "File"
1339                                     }
1340                                 ],
1341                                 "steps": [
1342                                     {
1343                                         "id": "#main/step1",
1344                                         "in": [
1345                                             {
1346                                                 "id": "#main/step1/pw",
1347                                                 "source": "#main/pw"
1348                                             }
1349                                         ],
1350                                         "out": [
1351                                             "#main/step1/out"
1352                                         ],
1353                                         "run": "#secret_job.cwl"
1354                                     }
1355                                 ]
1356                             }
1357                         ],
1358                         "cwlVersion": "v1.0"
1359                     },
1360                     "kind": "json"
1361                 },
1362                 "/var/spool/cwl": {
1363                     "kind": "collection",
1364                     "writable": True
1365                 },
1366                 "stdout": {
1367                     "kind": "file",
1368                     "path": "/var/spool/cwl/cwl.output.json"
1369                 }
1370             },
1371             "name": "secret_wf.cwl",
1372             "output_path": "/var/spool/cwl",
1373             "priority": 500,
1374             "properties": {},
1375             "runtime_constraints": {
1376                 "API": True,
1377                 "ram": 1342177280,
1378                 "vcpus": 1
1379             },
1380             "secret_mounts": {
1381                 "/secrets/s0": {
1382                     "content": "blorp",
1383                     "kind": "text"
1384                 }
1385             },
1386             "state": "Committed",
1387             "use_existing": True
1388         }
1389
1390         stubs.api.container_requests().create.assert_called_with(
1391             body=JsonDiffMatcher(expect_container))
1392         self.assertEqual(stubs.capture_stdout.getvalue(),
1393                          stubs.expect_container_request_uuid + '\n')
1394         self.assertEqual(exited, 0)
1395
1396     @stubs
1397     def test_submit_request_uuid(self, stubs):
1398         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1399
1400         stubs.api.container_requests().update().execute.return_value = {
1401             "uuid": stubs.expect_container_request_uuid,
1402             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1403             "state": "Queued"
1404         }
1405
1406         exited = arvados_cwl.main(
1407             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1408                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1410
1411         stubs.api.container_requests().update.assert_called_with(
1412             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1413         self.assertEqual(stubs.capture_stdout.getvalue(),
1414                          stubs.expect_container_request_uuid + '\n')
1415         self.assertEqual(exited, 0)
1416
1417     @stubs
1418     def test_submit_container_cluster_id(self, stubs):
1419         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1420
1421         exited = arvados_cwl.main(
1422             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1423                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1424             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1425
1426         expect_container = copy.deepcopy(stubs.expect_container_spec)
1427
1428         stubs.api.container_requests().create.assert_called_with(
1429             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1430         self.assertEqual(stubs.capture_stdout.getvalue(),
1431                          stubs.expect_container_request_uuid + '\n')
1432         self.assertEqual(exited, 0)
1433
1434     @stubs
1435     def test_submit_validate_cluster_id(self, stubs):
1436         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1437         exited = arvados_cwl.main(
1438             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1439              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1440             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1441         self.assertEqual(exited, 1)
1442
1443     @mock.patch("arvados.collection.CollectionReader")
1444     @stubs
1445     def test_submit_uuid_inputs(self, stubs, collectionReader):
1446         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1447         def list_side_effect(**kwargs):
1448             m = mock.MagicMock()
1449             if "count" in kwargs:
1450                 m.execute.return_value = {"items": [
1451                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1452                 ]}
1453             else:
1454                 m.execute.return_value = {"items": []}
1455             return m
1456         stubs.api.collections().list.side_effect = list_side_effect
1457
1458         exited = arvados_cwl.main(
1459             ["--submit", "--no-wait", "--api=containers", "--debug",
1460                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1461             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1462
1463         expect_container = copy.deepcopy(stubs.expect_container_spec)
1464         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1465         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1466         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1467
1468         stubs.api.collections().list.assert_has_calls([
1469             mock.call(count='none',
1470                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1471                       select=['uuid', 'portable_data_hash'])])
1472         stubs.api.container_requests().create.assert_called_with(
1473             body=JsonDiffMatcher(expect_container))
1474         self.assertEqual(stubs.capture_stdout.getvalue(),
1475                          stubs.expect_container_request_uuid + '\n')
1476         self.assertEqual(exited, 0)
1477
1478     @stubs
1479     def test_submit_mismatched_uuid_inputs(self, stubs):
1480         def list_side_effect(**kwargs):
1481             m = mock.MagicMock()
1482             if "count" in kwargs:
1483                 m.execute.return_value = {"items": [
1484                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1485                 ]}
1486             else:
1487                 m.execute.return_value = {"items": []}
1488             return m
1489         stubs.api.collections().list.side_effect = list_side_effect
1490
1491         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1492             capture_stderr = StringIO()
1493             cwltool_logger = logging.getLogger('cwltool')
1494             stderr_logger = logging.StreamHandler(capture_stderr)
1495             cwltool_logger.addHandler(stderr_logger)
1496
1497             try:
1498                 exited = arvados_cwl.main(
1499                     ["--submit", "--no-wait", "--api=containers", "--debug",
1500                         "tests/wf/submit_wf.cwl", infile],
1501                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1502
1503                 self.assertEqual(exited, 1)
1504                 self.assertRegexpMatches(
1505                     capture_stderr.getvalue(),
1506                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1507             finally:
1508                 cwltool_logger.removeHandler(stderr_logger)
1509
1510     @mock.patch("arvados.collection.CollectionReader")
1511     @stubs
1512     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1513         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1514         capture_stderr = StringIO()
1515
1516         cwltool_logger = logging.getLogger('cwltool')
1517         stderr_logger = logging.StreamHandler(capture_stderr)
1518         cwltool_logger.addHandler(stderr_logger)
1519
1520         exited = arvados_cwl.main(
1521             ["--submit", "--no-wait", "--api=containers", "--debug",
1522                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1523             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1524
1525         try:
1526             self.assertEqual(exited, 1)
1527             self.assertRegexpMatches(
1528                 capture_stderr.getvalue(),
1529                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1530         finally:
1531             cwltool_logger.removeHandler(stderr_logger)
1532
1533
1534 class TestCreateTemplate(unittest.TestCase):
1535     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1536
1537     def _adjust_script_params(self, expect_component):
1538         expect_component['script_parameters']['x'] = {
1539             'dataclass': 'File',
1540             'required': True,
1541             'type': 'File',
1542             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1543         }
1544         expect_component['script_parameters']['y'] = {
1545             'dataclass': 'Collection',
1546             'required': True,
1547             'type': 'Directory',
1548             'value': '99999999999999999999999999999998+99',
1549         }
1550         expect_component['script_parameters']['z'] = {
1551             'dataclass': 'Collection',
1552             'required': True,
1553             'type': 'Directory',
1554         }
1555
1556     @stubs
1557     def test_create(self, stubs):
1558         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1559
1560         exited = arvados_cwl.main(
1561             ["--create-workflow", "--debug",
1562              "--api=jobs",
1563              "--project-uuid", project_uuid,
1564              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1565             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1566
1567         stubs.api.pipeline_instances().create.refute_called()
1568         stubs.api.jobs().create.refute_called()
1569
1570         expect_component = copy.deepcopy(stubs.expect_job_spec)
1571         self._adjust_script_params(expect_component)
1572         expect_template = {
1573             "components": {
1574                 "submit_wf.cwl": expect_component,
1575             },
1576             "name": "submit_wf.cwl",
1577             "owner_uuid": project_uuid,
1578         }
1579         stubs.api.pipeline_templates().create.assert_called_with(
1580             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1581
1582         self.assertEqual(stubs.capture_stdout.getvalue(),
1583                          stubs.expect_pipeline_template_uuid + '\n')
1584         self.assertEqual(exited, 0)
1585
1586     @stubs
1587     def test_create_name(self, stubs):
1588         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1589
1590         exited = arvados_cwl.main(
1591             ["--create-workflow", "--debug",
1592              "--project-uuid", project_uuid,
1593              "--api=jobs",
1594              "--name", "testing 123",
1595              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1596             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1597
1598         stubs.api.pipeline_instances().create.refute_called()
1599         stubs.api.jobs().create.refute_called()
1600
1601         expect_component = copy.deepcopy(stubs.expect_job_spec)
1602         self._adjust_script_params(expect_component)
1603         expect_template = {
1604             "components": {
1605                 "testing 123": expect_component,
1606             },
1607             "name": "testing 123",
1608             "owner_uuid": project_uuid,
1609         }
1610         stubs.api.pipeline_templates().create.assert_called_with(
1611             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1612
1613         self.assertEqual(stubs.capture_stdout.getvalue(),
1614                          stubs.expect_pipeline_template_uuid + '\n')
1615         self.assertEqual(exited, 0)
1616
1617     @stubs
1618     def test_update_name(self, stubs):
1619         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1620
1621         exited = arvados_cwl.main(
1622             ["--update-workflow", self.existing_template_uuid,
1623              "--debug",
1624              "--project-uuid", project_uuid,
1625              "--api=jobs",
1626              "--name", "testing 123",
1627              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1628             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1629
1630         stubs.api.pipeline_instances().create.refute_called()
1631         stubs.api.jobs().create.refute_called()
1632
1633         expect_component = copy.deepcopy(stubs.expect_job_spec)
1634         self._adjust_script_params(expect_component)
1635         expect_template = {
1636             "components": {
1637                 "testing 123": expect_component,
1638             },
1639             "name": "testing 123",
1640             "owner_uuid": project_uuid,
1641         }
1642         stubs.api.pipeline_templates().create.refute_called()
1643         stubs.api.pipeline_templates().update.assert_called_with(
1644             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1645
1646         self.assertEqual(stubs.capture_stdout.getvalue(),
1647                          self.existing_template_uuid + '\n')
1648         self.assertEqual(exited, 0)
1649
1650
1651 class TestCreateWorkflow(unittest.TestCase):
1652     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1653     expect_workflow = StripYAMLComments(
1654         open("tests/wf/expect_packed.cwl").read())
1655
1656     @stubs
1657     def test_create(self, stubs):
1658         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1659
1660         exited = arvados_cwl.main(
1661             ["--create-workflow", "--debug",
1662              "--api=containers",
1663              "--project-uuid", project_uuid,
1664              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1665             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1666
1667         stubs.api.pipeline_templates().create.refute_called()
1668         stubs.api.container_requests().create.refute_called()
1669
1670         body = {
1671             "workflow": {
1672                 "owner_uuid": project_uuid,
1673                 "name": "submit_wf.cwl",
1674                 "description": "",
1675                 "definition": self.expect_workflow,
1676             }
1677         }
1678         stubs.api.workflows().create.assert_called_with(
1679             body=JsonDiffMatcher(body))
1680
1681         self.assertEqual(stubs.capture_stdout.getvalue(),
1682                          stubs.expect_workflow_uuid + '\n')
1683         self.assertEqual(exited, 0)
1684
1685     @stubs
1686     def test_create_name(self, stubs):
1687         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1688
1689         exited = arvados_cwl.main(
1690             ["--create-workflow", "--debug",
1691              "--api=containers",
1692              "--project-uuid", project_uuid,
1693              "--name", "testing 123",
1694              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1695             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1696
1697         stubs.api.pipeline_templates().create.refute_called()
1698         stubs.api.container_requests().create.refute_called()
1699
1700         body = {
1701             "workflow": {
1702                 "owner_uuid": project_uuid,
1703                 "name": "testing 123",
1704                 "description": "",
1705                 "definition": self.expect_workflow,
1706             }
1707         }
1708         stubs.api.workflows().create.assert_called_with(
1709             body=JsonDiffMatcher(body))
1710
1711         self.assertEqual(stubs.capture_stdout.getvalue(),
1712                          stubs.expect_workflow_uuid + '\n')
1713         self.assertEqual(exited, 0)
1714
1715     @stubs
1716     def test_incompatible_api(self, stubs):
1717         capture_stderr = StringIO()
1718         acr_logger = logging.getLogger('arvados.cwl-runner')
1719         stderr_logger = logging.StreamHandler(capture_stderr)
1720         acr_logger.addHandler(stderr_logger)
1721
1722         try:
1723             exited = arvados_cwl.main(
1724                 ["--update-workflow", self.existing_workflow_uuid,
1725                  "--api=jobs",
1726                  "--debug",
1727                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1728                 sys.stderr, sys.stderr, api_client=stubs.api)
1729             self.assertEqual(exited, 1)
1730             self.assertRegexpMatches(
1731                 capture_stderr.getvalue(),
1732                 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1733         finally:
1734             acr_logger.removeHandler(stderr_logger)
1735
1736     @stubs
1737     def test_update(self, stubs):
1738         exited = arvados_cwl.main(
1739             ["--update-workflow", self.existing_workflow_uuid,
1740              "--debug",
1741              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1742             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1743
1744         body = {
1745             "workflow": {
1746                 "name": "submit_wf.cwl",
1747                 "description": "",
1748                 "definition": self.expect_workflow,
1749             }
1750         }
1751         stubs.api.workflows().update.assert_called_with(
1752             uuid=self.existing_workflow_uuid,
1753             body=JsonDiffMatcher(body))
1754         self.assertEqual(stubs.capture_stdout.getvalue(),
1755                          self.existing_workflow_uuid + '\n')
1756         self.assertEqual(exited, 0)
1757
1758     @stubs
1759     def test_update_name(self, stubs):
1760         exited = arvados_cwl.main(
1761             ["--update-workflow", self.existing_workflow_uuid,
1762              "--debug", "--name", "testing 123",
1763              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1764             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1765
1766         body = {
1767             "workflow": {
1768                 "name": "testing 123",
1769                 "description": "",
1770                 "definition": self.expect_workflow,
1771             }
1772         }
1773         stubs.api.workflows().update.assert_called_with(
1774             uuid=self.existing_workflow_uuid,
1775             body=JsonDiffMatcher(body))
1776         self.assertEqual(stubs.capture_stdout.getvalue(),
1777                          self.existing_workflow_uuid + '\n')
1778         self.assertEqual(exited, 0)
1779
1780     @stubs
1781     def test_create_collection_per_tool(self, stubs):
1782         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1783
1784         exited = arvados_cwl.main(
1785             ["--create-workflow", "--debug",
1786              "--api=containers",
1787              "--project-uuid", project_uuid,
1788              "tests/collection_per_tool/collection_per_tool.cwl"],
1789             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1790
1791         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1792         expect_workflow = StripYAMLComments(open(toolfile).read())
1793
1794         body = {
1795             "workflow": {
1796                 "owner_uuid": project_uuid,
1797                 "name": "collection_per_tool.cwl",
1798                 "description": "",
1799                 "definition": expect_workflow,
1800             }
1801         }
1802         stubs.api.workflows().create.assert_called_with(
1803             body=JsonDiffMatcher(body))
1804
1805         self.assertEqual(stubs.capture_stdout.getvalue(),
1806                          stubs.expect_workflow_uuid + '\n')
1807         self.assertEqual(exited, 0)
1808
1809 class TestTemplateInputs(unittest.TestCase):
1810     expect_template = {
1811         "components": {
1812             "inputs_test.cwl": {
1813                 'runtime_constraints': {
1814                     'docker_image': '999999999999999999999999999999d3+99',
1815                     'min_ram_mb_per_node': 1024
1816                 },
1817                 'script_parameters': {
1818                     'cwl:tool':
1819                     'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
1820                     'optionalFloatInput': None,
1821                     'fileInput': {
1822                         'type': 'File',
1823                         'dataclass': 'File',
1824                         'required': True,
1825                         'title': "It's a file; we expect to find some characters in it.",
1826                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1827                     },
1828                     'floatInput': {
1829                         'type': 'float',
1830                         'dataclass': 'number',
1831                         'required': True,
1832                         'title': 'Floats like a duck',
1833                         'default': 0.1,
1834                         'value': 0.1,
1835                     },
1836                     'optionalFloatInput': {
1837                         'type': ['null', 'float'],
1838                         'dataclass': 'number',
1839                         'required': False,
1840                     },
1841                     'boolInput': {
1842                         'type': 'boolean',
1843                         'dataclass': 'boolean',
1844                         'required': True,
1845                         'title': 'True or false?',
1846                     },
1847                 },
1848                 'repository': 'arvados',
1849                 'script_version': 'master',
1850                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1851                 'script': 'cwl-runner',
1852             },
1853         },
1854         "name": "inputs_test.cwl",
1855     }
1856
1857     @stubs
1858     def test_inputs_empty(self, stubs):
1859         exited = arvados_cwl.main(
1860             ["--create-template",
1861              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1862             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1863
1864         stubs.api.pipeline_templates().create.assert_called_with(
1865             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1866
1867         self.assertEqual(exited, 0)
1868
1869     @stubs
1870     def test_inputs(self, stubs):
1871         exited = arvados_cwl.main(
1872             ["--create-template",
1873              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1874             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1875
1876         expect_template = copy.deepcopy(self.expect_template)
1877         params = expect_template[
1878             "components"]["inputs_test.cwl"]["script_parameters"]
1879         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1880         params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
1881         params["floatInput"]["value"] = 1.234
1882         params["boolInput"]["value"] = True
1883
1884         stubs.api.pipeline_templates().create.assert_called_with(
1885             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1886         self.assertEqual(exited, 0)