sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20
  21 from io import BytesIO
  22
  23 # StringIO.StringIO and io.StringIO have different behavior write() is
  24 # called with both python2 (byte) strings and unicode strings
  25 # (specifically there's some logging in cwltool that causes trouble).
  26 # This isn't a problem on python3 because all string are unicode.
  27 if sys.version_info[0] < 3:
  28     from StringIO import StringIO
  29 else:
  30     from io import StringIO
  31
  32 import arvados
  33 import arvados.collection
  34 import arvados_cwl
  35 import arvados_cwl.executor
  36 import arvados_cwl.runner
  37 import arvados.keep
  38
  39 from .matcher import JsonDiffMatcher, StripYAMLComments
  40 from .mock_discovery import get_rootDesc
  41
  42 import ruamel.yaml as yaml
  43
  44 _rootDesc = None
  45
  46 def stubs(func):
  47     @functools.wraps(func)
  48     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  49     @mock.patch("arvados.collection.KeepClient")
  50     @mock.patch("arvados.keep.KeepClient")
  51     @mock.patch("arvados.events.subscribe")
  52     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  53         class Stubs(object):
  54             pass
  55         stubs = Stubs()
  56         stubs.events = events
  57         stubs.keepdocker = keepdocker
  58
  59         def putstub(p, **kwargs):
  60             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  61         keep_client1().put.side_effect = putstub
  62         keep_client1.put.side_effect = putstub
  63         keep_client2().put.side_effect = putstub
  64         keep_client2.put.side_effect = putstub
  65
  66         stubs.keep_client = keep_client2
  67         stubs.docker_images = {
  68             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  69             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  70             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  71             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  72         }
  73         def kd(a, b, image_name=None, image_tag=None):
  74             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  75         stubs.keepdocker.side_effect = kd
  76
  77         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  78         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  79
  80         if sys.version_info[0] < 3:
  81             stubs.capture_stdout = BytesIO()
  82         else:
  83             stubs.capture_stdout = StringIO()
  84
  85         stubs.api = mock.MagicMock()
  86         stubs.api._rootDesc = get_rootDesc()
  87
  88         stubs.api.users().current().execute.return_value = {
  89             "uuid": stubs.fake_user_uuid,
  90         }
  91         stubs.api.collections().list().execute.return_value = {"items": []}
  92         stubs.api.containers().current().execute.return_value = {
  93             "uuid": stubs.fake_container_uuid,
  94         }
  95
  96         class CollectionExecute(object):
  97             def __init__(self, exe):
  98                 self.exe = exe
  99             def execute(self, num_retries=None):
 100                 return self.exe
 101
 102         def collection_createstub(created_collections, body, ensure_unique_name=None):
 103             mt = body["manifest_text"].encode('utf-8')
 104             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 105             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 106             created_collections[uuid] = {
 107                 "uuid": uuid,
 108                 "portable_data_hash": pdh,
 109                 "manifest_text": mt.decode('utf-8')
 110             }
 111             return CollectionExecute(created_collections[uuid])
 112
 113         def collection_getstub(created_collections, uuid):
 114             for v in viewvalues(created_collections):
 115                 if uuid in (v["uuid"], v["portable_data_hash"]):
 116                     return CollectionExecute(v)
 117
 118         created_collections = {
 119             "99999999999999999999999999999998+99": {
 120                 "uuid": "",
 121                 "portable_data_hash": "99999999999999999999999999999998+99",
 122                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 123             },
 124             "99999999999999999999999999999997+99": {
 125                 "uuid": "",
 126                 "portable_data_hash": "99999999999999999999999999999997+99",
 127                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 128             },
 129             "99999999999999999999999999999994+99": {
 130                 "uuid": "",
 131                 "portable_data_hash": "99999999999999999999999999999994+99",
 132                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 133             },
 134             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 135                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 136                 "portable_data_hash": "999999999999999999999999999999d3+99",
 137                 "manifest_text": ""
 138             },
 139             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 140                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 141                 "portable_data_hash": "999999999999999999999999999999d4+99",
 142                 "manifest_text": ""
 143             },
 144             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 145                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 146                 "portable_data_hash": "999999999999999999999999999999d5+99",
 147                 "manifest_text": ""
 148             },
 149             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 150                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 151                 "portable_data_hash": "999999999999999999999999999999d6+99",
 152                 "manifest_text": ""
 153             }
 154         }
 155         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 156         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 157
 158         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 159         stubs.api.jobs().create().execute.return_value = {
 160             "uuid": stubs.expect_job_uuid,
 161             "state": "Queued",
 162         }
 163
 164         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 165         stubs.api.container_requests().create().execute.return_value = {
 166             "uuid": stubs.expect_container_request_uuid,
 167             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 168             "state": "Queued"
 169         }
 170
 171         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 172         stubs.api.pipeline_templates().create().execute.return_value = {
 173             "uuid": stubs.expect_pipeline_template_uuid,
 174         }
 175         stubs.expect_job_spec = {
 176             'runtime_constraints': {
 177                 'docker_image': '999999999999999999999999999999d3+99',
 178                 'min_ram_mb_per_node': 1024
 179             },
 180             'script_parameters': {
 181                 'x': {
 182                     'basename': 'blorp.txt',
 183                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 184                     'class': 'File'
 185                 },
 186                 'y': {
 187                     'basename': '99999999999999999999999999999998+99',
 188                     'location': 'keep:99999999999999999999999999999998+99',
 189                     'class': 'Directory'
 190                 },
 191                 'z': {
 192                     'basename': 'anonymous',
 193                     "listing": [{
 194                         "basename": "renamed.txt",
 195                         "class": "File",
 196                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 197                         "size": 0
 198                     }],
 199                     'class': 'Directory'
 200                 },
 201                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 202             },
 203             'repository': 'arvados',
 204             'script_version': 'master',
 205             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 206             'script': 'cwl-runner'
 207         }
 208         stubs.pipeline_component = stubs.expect_job_spec.copy()
 209         stubs.expect_pipeline_instance = {
 210             'name': 'submit_wf.cwl',
 211             'state': 'RunningOnServer',
 212             'owner_uuid': None,
 213             "components": {
 214                 "cwl-runner": {
 215                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 216                     'script_parameters': {
 217                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 218                         'x': {"value": {
 219                             'basename': 'blorp.txt',
 220                             'class': 'File',
 221                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         }},
 224                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 225                               'listing': [
 226                                   {
 227                                       'basename': 'renamed.txt',
 228                                       'class': 'File', 'location':
 229                                       'keep:99999999999999999999999999999998+99/file1.txt',
 230                                       'size': 0
 231                                   }
 232                               ]}},
 233                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 234                         'arv:debug': True,
 235                         'arv:enable_reuse': True,
 236                         'arv:on_error': 'continue'
 237                     },
 238                     'repository': 'arvados',
 239                     'script_version': 'master',
 240                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 241                     'script': 'cwl-runner',
 242                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 243                 }
 244             }
 245         }
 246         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 247         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 248         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 249         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 250         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 251             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 252             "state": "Queued"
 253         }
 254         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 255         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 256
 257         with open("tests/wf/submit_wf_packed.cwl") as f:
 258             expect_packed_workflow = yaml.round_trip_load(f)
 259
 260         stubs.expect_container_spec = {
 261             'priority': 500,
 262             'mounts': {
 263                 '/var/spool/cwl': {
 264                     'writable': True,
 265                     'kind': 'collection'
 266                 },
 267                 '/var/lib/cwl/workflow.json': {
 268                     'content': expect_packed_workflow,
 269                     'kind': 'json'
 270                 },
 271                 'stdout': {
 272                     'path': '/var/spool/cwl/cwl.output.json',
 273                     'kind': 'file'
 274                 },
 275                 '/var/lib/cwl/cwl.input.json': {
 276                     'kind': 'json',
 277                     'content': {
 278                         'y': {
 279                             'basename': '99999999999999999999999999999998+99',
 280                             'location': 'keep:99999999999999999999999999999998+99',
 281                             'class': 'Directory'},
 282                         'x': {
 283                             'basename': u'blorp.txt',
 284                             'class': 'File',
 285                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 286                             "size": 16
 287                         },
 288                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 289                             {'basename': 'renamed.txt',
 290                              'class': 'File',
 291                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 292                              'size': 0
 293                             }
 294                         ]}
 295                     },
 296                     'kind': 'json'
 297                 }
 298             },
 299             'secret_mounts': {},
 300             'state': 'Committed',
 301             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 302                         '--no-log-timestamps', '--disable-validate',
 303                         '--eval-timeout=20', '--thread-count=1',
 304                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 305                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 306             'name': 'submit_wf.cwl',
 307             'container_image': '999999999999999999999999999999d3+99',
 308             'output_path': '/var/spool/cwl',
 309             'cwd': '/var/spool/cwl',
 310             'runtime_constraints': {
 311                 'API': True,
 312                 'vcpus': 1,
 313                 'ram': (1024+256)*1024*1024
 314             },
 315             'use_existing': True,
 316             'properties': {},
 317             'secret_mounts': {}
 318         }
 319
 320         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 321         stubs.api.workflows().create().execute.return_value = {
 322             "uuid": stubs.expect_workflow_uuid,
 323         }
 324         def update_mock(**kwargs):
 325             stubs.updated_uuid = kwargs.get('uuid')
 326             return mock.DEFAULT
 327         stubs.api.workflows().update.side_effect = update_mock
 328         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 329             "uuid": stubs.updated_uuid,
 330         }
 331
 332         return func(self, stubs, *args, **kwargs)
 333     return wrapped
 334
 335
 336 class TestSubmit(unittest.TestCase):
 337     @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
 338     @mock.patch("time.sleep")
 339     @stubs
 340     def test_submit(self, stubs, tm, arvdock):
 341         def get_image(api_client, dockerRequirement, pull_image, project_uuid):
 342             if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
 343                 return '999999999999999999999999999999d3+99'
 344             elif dockerRequirement["dockerPull"] == "debian:8":
 345                 return '999999999999999999999999999999d4+99'
 346         arvdock.side_effect = get_image
 347
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--api=jobs", "--debug",
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 352
 353         stubs.api.collections().create.assert_has_calls([
 354             mock.call(body=JsonDiffMatcher({
 355                 'manifest_text':
 356                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 357                 'replication_desired': None,
 358                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 359             }), ensure_unique_name=False),
 360             mock.call(body=JsonDiffMatcher({
 361                 'manifest_text':
 362                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 363                 'replication_desired': None,
 364                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 365             }), ensure_unique_name=False),
 366             mock.call(body=JsonDiffMatcher({
 367                 'manifest_text':
 368                 ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
 369                 'replication_desired': None,
 370                 'name': 'submit_wf.cwl',
 371             }), ensure_unique_name=True)        ])
 372
 373         arvdock.assert_has_calls([
 374             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 375             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
 376             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 377         ])
 378
 379         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 380         stubs.api.pipeline_instances().create.assert_called_with(
 381             body=JsonDiffMatcher(expect_pipeline))
 382         self.assertEqual(stubs.capture_stdout.getvalue(),
 383                          stubs.expect_pipeline_uuid + '\n')
 384         self.assertEqual(exited, 0)
 385
 386     @mock.patch("time.sleep")
 387     @stubs
 388     def test_submit_no_reuse(self, stubs, tm):
 389         exited = arvados_cwl.main(
 390             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 391              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 392             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 393
 394         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 395         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 396         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 397
 398         stubs.api.pipeline_instances().create.assert_called_with(
 399             body=JsonDiffMatcher(expect_pipeline))
 400         self.assertEqual(stubs.capture_stdout.getvalue(),
 401                          stubs.expect_pipeline_uuid + '\n')
 402         self.assertEqual(exited, 0)
 403
 404     @stubs
 405     def test_error_when_multiple_storage_classes_specified(self, stubs):
 406         storage_classes = "foo,bar"
 407         exited = arvados_cwl.main(
 408                 ["--debug", "--storage-classes", storage_classes,
 409                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410                 sys.stdin, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 1)
 412
 413     @mock.patch("time.sleep")
 414     @stubs
 415     def test_submit_on_error(self, stubs, tm):
 416         exited = arvados_cwl.main(
 417             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 418              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 419             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 420
 421         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 422         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 423
 424         stubs.api.pipeline_instances().create.assert_called_with(
 425             body=JsonDiffMatcher(expect_pipeline))
 426         self.assertEqual(stubs.capture_stdout.getvalue(),
 427                          stubs.expect_pipeline_uuid + '\n')
 428         self.assertEqual(exited, 0)
 429
 430     @mock.patch("time.sleep")
 431     @stubs
 432     def test_submit_runner_ram(self, stubs, tm):
 433         exited = arvados_cwl.main(
 434             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 435              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 436             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 437
 438         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 439         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 440
 441         stubs.api.pipeline_instances().create.assert_called_with(
 442             body=JsonDiffMatcher(expect_pipeline))
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_pipeline_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @mock.patch("time.sleep")
 448     @stubs
 449     def test_submit_invalid_runner_ram(self, stubs, tm):
 450         exited = arvados_cwl.main(
 451             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 452              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 453             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 454         self.assertEqual(exited, 1)
 455
 456     @mock.patch("time.sleep")
 457     @stubs
 458     def test_submit_output_name(self, stubs, tm):
 459         output_name = "test_output_name"
 460
 461         exited = arvados_cwl.main(
 462             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 463              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 464             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 465
 466         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 467         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 468
 469         stubs.api.pipeline_instances().create.assert_called_with(
 470             body=JsonDiffMatcher(expect_pipeline))
 471         self.assertEqual(stubs.capture_stdout.getvalue(),
 472                          stubs.expect_pipeline_uuid + '\n')
 473         self.assertEqual(exited, 0)
 474
 475     @mock.patch("time.sleep")
 476     @stubs
 477     def test_submit_pipeline_name(self, stubs, tm):
 478         exited = arvados_cwl.main(
 479             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 480              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 481             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 482         self.assertEqual(exited, 0)
 483
 484         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 485         expect_pipeline["name"] = "hello job 123"
 486
 487         stubs.api.pipeline_instances().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_pipeline))
 489         self.assertEqual(stubs.capture_stdout.getvalue(),
 490                          stubs.expect_pipeline_uuid + '\n')
 491
 492     @mock.patch("time.sleep")
 493     @stubs
 494     def test_submit_output_tags(self, stubs, tm):
 495         output_tags = "tag0,tag1,tag2"
 496
 497         exited = arvados_cwl.main(
 498             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 499              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 500             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 501         self.assertEqual(exited, 0)
 502
 503         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 504         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 505
 506         stubs.api.pipeline_instances().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_pipeline))
 508         self.assertEqual(stubs.capture_stdout.getvalue(),
 509                          stubs.expect_pipeline_uuid + '\n')
 510
 511     @mock.patch("time.sleep")
 512     @stubs
 513     def test_submit_with_project_uuid(self, stubs, tm):
 514         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--debug",
 518              "--project-uuid", project_uuid,
 519              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 520             sys.stdout, sys.stderr, api_client=stubs.api)
 521         self.assertEqual(exited, 0)
 522
 523         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 524         expect_pipeline["owner_uuid"] = project_uuid
 525         stubs.api.pipeline_instances().create.assert_called_with(
 526             body=JsonDiffMatcher(expect_pipeline))
 527
 528     @stubs
 529     def test_submit_container(self, stubs):
 530         exited = arvados_cwl.main(
 531             ["--submit", "--no-wait", "--api=containers", "--debug",
 532                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 533             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 534
 535         stubs.api.collections().create.assert_has_calls([
 536             mock.call(body=JsonDiffMatcher({
 537                 'manifest_text':
 538                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 539                 'replication_desired': None,
 540                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 541             }), ensure_unique_name=False),
 542             mock.call(body=JsonDiffMatcher({
 543                 'manifest_text':
 544                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 545                 'replication_desired': None,
 546                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 547             }), ensure_unique_name=False)])
 548
 549         expect_container = copy.deepcopy(stubs.expect_container_spec)
 550         stubs.api.container_requests().create.assert_called_with(
 551             body=JsonDiffMatcher(expect_container))
 552         self.assertEqual(stubs.capture_stdout.getvalue(),
 553                          stubs.expect_container_request_uuid + '\n')
 554         self.assertEqual(exited, 0)
 555
 556     @stubs
 557     def test_submit_container_no_reuse(self, stubs):
 558         exited = arvados_cwl.main(
 559             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 560                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 561             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 562
 563         expect_container = copy.deepcopy(stubs.expect_container_spec)
 564         expect_container["command"] = [
 565             'arvados-cwl-runner', '--local', '--api=containers',
 566             '--no-log-timestamps', '--disable-validate',
 567             '--eval-timeout=20', '--thread-count=1',
 568             '--disable-reuse', "--collection-cache-size=256",
 569             '--debug', '--on-error=continue',
 570             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 571         expect_container["use_existing"] = False
 572
 573         stubs.api.container_requests().create.assert_called_with(
 574             body=JsonDiffMatcher(expect_container))
 575         self.assertEqual(stubs.capture_stdout.getvalue(),
 576                          stubs.expect_container_request_uuid + '\n')
 577         self.assertEqual(exited, 0)
 578
 579     @stubs
 580     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 581         exited = arvados_cwl.main(
 582             ["--submit", "--no-wait", "--api=containers", "--debug",
 583              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 584             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 585         self.assertEqual(exited, 0)
 586
 587         expect_container = copy.deepcopy(stubs.expect_container_spec)
 588         expect_container["command"] = [
 589             'arvados-cwl-runner', '--local', '--api=containers',
 590             '--no-log-timestamps', '--disable-validate',
 591             '--eval-timeout=20', '--thread-count=1',
 592             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 593             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 594         expect_container["use_existing"] = False
 595         expect_container["name"] = "submit_wf_no_reuse.cwl"
 596         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 597             {
 598                 "class": "http://arvados.org/cwl#ReuseRequirement",
 599                 "enableReuse": False,
 600             },
 601         ]
 602         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 603             "arv": "http://arvados.org/cwl#",
 604             "cwltool": "http://commonwl.org/cwltool#"
 605         }
 606
 607         stubs.api.container_requests().create.assert_called_with(
 608             body=JsonDiffMatcher(expect_container))
 609         self.assertEqual(stubs.capture_stdout.getvalue(),
 610                          stubs.expect_container_request_uuid + '\n')
 611
 612
 613     @stubs
 614     def test_submit_container_on_error(self, stubs):
 615         exited = arvados_cwl.main(
 616             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 617                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 618             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 619
 620         expect_container = copy.deepcopy(stubs.expect_container_spec)
 621         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 622                                        '--no-log-timestamps', '--disable-validate',
 623                                        '--eval-timeout=20', '--thread-count=1',
 624                                        '--enable-reuse', "--collection-cache-size=256",
 625                                        '--debug', '--on-error=stop',
 626                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(stubs.capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632         self.assertEqual(exited, 0)
 633
 634     @stubs
 635     def test_submit_container_output_name(self, stubs):
 636         output_name = "test_output_name"
 637
 638         exited = arvados_cwl.main(
 639             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 640                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 641             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 642
 643         expect_container = copy.deepcopy(stubs.expect_container_spec)
 644         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 645                                        '--no-log-timestamps', '--disable-validate',
 646                                        '--eval-timeout=20', '--thread-count=1',
 647                                        '--enable-reuse', "--collection-cache-size=256",
 648                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 649                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 650         expect_container["output_name"] = output_name
 651
 652         stubs.api.container_requests().create.assert_called_with(
 653             body=JsonDiffMatcher(expect_container))
 654         self.assertEqual(stubs.capture_stdout.getvalue(),
 655                          stubs.expect_container_request_uuid + '\n')
 656         self.assertEqual(exited, 0)
 657
 658     @stubs
 659     def test_submit_storage_classes(self, stubs):
 660         exited = arvados_cwl.main(
 661             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 662                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 663             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 664
 665         expect_container = copy.deepcopy(stubs.expect_container_spec)
 666         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 667                                        '--no-log-timestamps', '--disable-validate',
 668                                        '--eval-timeout=20', '--thread-count=1',
 669                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 670                                        "--storage-classes=foo", '--on-error=continue',
 671                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 672
 673         stubs.api.container_requests().create.assert_called_with(
 674             body=JsonDiffMatcher(expect_container))
 675         self.assertEqual(stubs.capture_stdout.getvalue(),
 676                          stubs.expect_container_request_uuid + '\n')
 677         self.assertEqual(exited, 0)
 678
 679     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 680     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 681     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 682     @stubs
 683     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 684         def set_final_output(job_order, output_callback, runtimeContext):
 685             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 686             return []
 687         job.side_effect = set_final_output
 688
 689         exited = arvados_cwl.main(
 690             ["--debug", "--local", "--storage-classes=foo",
 691                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 692             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 693
 694         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 695         self.assertEqual(exited, 0)
 696
 697     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 698     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 699     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 700     @stubs
 701     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 702         def set_final_output(job_order, output_callback, runtimeContext):
 703             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 704             return []
 705         job.side_effect = set_final_output
 706
 707         exited = arvados_cwl.main(
 708             ["--debug", "--local",
 709                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 710             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 711
 712         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 713         self.assertEqual(exited, 0)
 714
 715     @stubs
 716     def test_submit_container_output_ttl(self, stubs):
 717         exited = arvados_cwl.main(
 718             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 719                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 720             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 721
 722         expect_container = copy.deepcopy(stubs.expect_container_spec)
 723         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 724                                        '--no-log-timestamps', '--disable-validate',
 725                                        '--eval-timeout=20', '--thread-count=1',
 726                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 727                                        '--on-error=continue',
 728                                        "--intermediate-output-ttl=3600",
 729                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 730
 731         stubs.api.container_requests().create.assert_called_with(
 732             body=JsonDiffMatcher(expect_container))
 733         self.assertEqual(stubs.capture_stdout.getvalue(),
 734                          stubs.expect_container_request_uuid + '\n')
 735         self.assertEqual(exited, 0)
 736
 737     @stubs
 738     def test_submit_container_trash_intermediate(self, stubs):
 739         exited = arvados_cwl.main(
 740             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 741                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 742             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 743
 744
 745         expect_container = copy.deepcopy(stubs.expect_container_spec)
 746         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 747                                        '--no-log-timestamps', '--disable-validate',
 748                                        '--eval-timeout=20', '--thread-count=1',
 749                                        '--enable-reuse', "--collection-cache-size=256",
 750                                        '--debug', '--on-error=continue',
 751                                        "--trash-intermediate",
 752                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 753
 754         stubs.api.container_requests().create.assert_called_with(
 755             body=JsonDiffMatcher(expect_container))
 756         self.assertEqual(stubs.capture_stdout.getvalue(),
 757                          stubs.expect_container_request_uuid + '\n')
 758         self.assertEqual(exited, 0)
 759
 760     @stubs
 761     def test_submit_container_output_tags(self, stubs):
 762         output_tags = "tag0,tag1,tag2"
 763
 764         exited = arvados_cwl.main(
 765             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 766                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 767             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 768
 769         expect_container = copy.deepcopy(stubs.expect_container_spec)
 770         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 771                                        '--no-log-timestamps', '--disable-validate',
 772                                        '--eval-timeout=20', '--thread-count=1',
 773                                        '--enable-reuse', "--collection-cache-size=256",
 774                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 775                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 776
 777         stubs.api.container_requests().create.assert_called_with(
 778             body=JsonDiffMatcher(expect_container))
 779         self.assertEqual(stubs.capture_stdout.getvalue(),
 780                          stubs.expect_container_request_uuid + '\n')
 781         self.assertEqual(exited, 0)
 782
 783     @stubs
 784     def test_submit_container_runner_ram(self, stubs):
 785         exited = arvados_cwl.main(
 786             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 787                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 788             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 789
 790         expect_container = copy.deepcopy(stubs.expect_container_spec)
 791         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 792
 793         stubs.api.container_requests().create.assert_called_with(
 794             body=JsonDiffMatcher(expect_container))
 795         self.assertEqual(stubs.capture_stdout.getvalue(),
 796                          stubs.expect_container_request_uuid + '\n')
 797         self.assertEqual(exited, 0)
 798
 799     @mock.patch("arvados.collection.CollectionReader")
 800     @mock.patch("time.sleep")
 801     @stubs
 802     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 803         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 804         exited = arvados_cwl.main(
 805             ["--submit", "--no-wait", "--api=containers", "--debug",
 806              "tests/wf/submit_keepref_wf.cwl"],
 807             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 808         self.assertEqual(exited, 0)
 809
 810     @mock.patch("arvados.collection.CollectionReader")
 811     @mock.patch("time.sleep")
 812     @stubs
 813     def test_submit_keepref(self, stubs, tm, reader):
 814         with open("tests/wf/expect_arvworkflow.cwl") as f:
 815             reader().open().__enter__().read.return_value = f.read()
 816
 817         exited = arvados_cwl.main(
 818             ["--submit", "--no-wait", "--api=containers", "--debug",
 819              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 820             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 821
 822         expect_container = {
 823             'priority': 500,
 824             'mounts': {
 825                 '/var/spool/cwl': {
 826                     'writable': True,
 827                     'kind': 'collection'
 828                 },
 829                 'stdout': {
 830                     'path': '/var/spool/cwl/cwl.output.json',
 831                     'kind': 'file'
 832                 },
 833                 '/var/lib/cwl/workflow': {
 834                     'portable_data_hash': '99999999999999999999999999999994+99',
 835                     'kind': 'collection'
 836                 },
 837                 '/var/lib/cwl/cwl.input.json': {
 838                     'content': {
 839                         'x': 'XxX'
 840                     },
 841                     'kind': 'json'
 842                 }
 843             }, 'state': 'Committed',
 844             'output_path': '/var/spool/cwl',
 845             'name': 'expect_arvworkflow.cwl#main',
 846             'container_image': '999999999999999999999999999999d3+99',
 847             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 848                         '--no-log-timestamps', '--disable-validate',
 849                         '--eval-timeout=20', '--thread-count=1',
 850                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 851                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 852             'cwd': '/var/spool/cwl',
 853             'runtime_constraints': {
 854                 'API': True,
 855                 'vcpus': 1,
 856                 'ram': 1342177280
 857             },
 858             'use_existing': True,
 859             'properties': {},
 860             'secret_mounts': {}
 861         }
 862
 863         stubs.api.container_requests().create.assert_called_with(
 864             body=JsonDiffMatcher(expect_container))
 865         self.assertEqual(stubs.capture_stdout.getvalue(),
 866                          stubs.expect_container_request_uuid + '\n')
 867         self.assertEqual(exited, 0)
 868
 869     @mock.patch("arvados.collection.CollectionReader")
 870     @mock.patch("time.sleep")
 871     @stubs
 872     def test_submit_jobs_keepref(self, stubs, tm, reader):
 873         with open("tests/wf/expect_arvworkflow.cwl") as f:
 874             reader().open().__enter__().read.return_value = f.read()
 875
 876         exited = arvados_cwl.main(
 877             ["--submit", "--no-wait", "--api=jobs", "--debug",
 878              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 879             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 880
 881         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 882         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 883         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 884         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 885         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 886         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 887         stubs.api.pipeline_instances().create.assert_called_with(
 888             body=JsonDiffMatcher(expect_pipeline))
 889         self.assertEqual(exited, 0)
 890
 891     @mock.patch("time.sleep")
 892     @stubs
 893     def test_submit_arvworkflow(self, stubs, tm):
 894         with open("tests/wf/expect_arvworkflow.cwl") as f:
 895             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 896
 897         exited = arvados_cwl.main(
 898             ["--submit", "--no-wait", "--api=containers", "--debug",
 899              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 900             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 901
 902         expect_container = {
 903             'priority': 500,
 904             'mounts': {
 905                 '/var/spool/cwl': {
 906                     'writable': True,
 907                     'kind': 'collection'
 908                 },
 909                 'stdout': {
 910                     'path': '/var/spool/cwl/cwl.output.json',
 911                     'kind': 'file'
 912                 },
 913                 '/var/lib/cwl/workflow.json': {
 914                     'kind': 'json',
 915                     'content': {
 916                         'cwlVersion': 'v1.0',
 917                         '$graph': [
 918                             {
 919                                 'id': '#main',
 920                                 'inputs': [
 921                                     {'type': 'string', 'id': '#main/x'}
 922                                 ],
 923                                 'steps': [
 924                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 925                                      'run': '#submit_tool.cwl',
 926                                      'id': '#main/step1',
 927                                      'out': []}
 928                                 ],
 929                                 'class': 'Workflow',
 930                                 'outputs': []
 931                             },
 932                             {
 933                                 'inputs': [
 934                                     {
 935                                         'inputBinding': {'position': 1},
 936                                         'type': 'string',
 937                                         'id': '#submit_tool.cwl/x'}
 938                                 ],
 939                                 'requirements': [
 940                                     {
 941                                         'dockerPull': 'debian:8',
 942                                         'class': 'DockerRequirement',
 943                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 944                                     }
 945                                 ],
 946                                 'id': '#submit_tool.cwl',
 947                                 'outputs': [],
 948                                 'baseCommand': 'cat',
 949                                 'class': 'CommandLineTool'
 950                             }
 951                         ]
 952                     }
 953                 },
 954                 '/var/lib/cwl/cwl.input.json': {
 955                     'content': {
 956                         'x': 'XxX'
 957                     },
 958                     'kind': 'json'
 959                 }
 960             }, 'state': 'Committed',
 961             'output_path': '/var/spool/cwl',
 962             'name': 'a test workflow',
 963             'container_image': "999999999999999999999999999999d3+99",
 964             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 965                         '--no-log-timestamps', '--disable-validate',
 966                         '--eval-timeout=20', '--thread-count=1',
 967                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 968                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 969             'cwd': '/var/spool/cwl',
 970             'runtime_constraints': {
 971                 'API': True,
 972                 'vcpus': 1,
 973                 'ram': 1342177280
 974             },
 975             'use_existing': True,
 976             'properties': {
 977                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 978             },
 979             'secret_mounts': {}
 980         }
 981
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     @stubs
 989     def test_submit_container_name(self, stubs):
 990         exited = arvados_cwl.main(
 991             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 992                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 993             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 994
 995         expect_container = copy.deepcopy(stubs.expect_container_spec)
 996         expect_container["name"] = "hello container 123"
 997
 998         stubs.api.container_requests().create.assert_called_with(
 999             body=JsonDiffMatcher(expect_container))
1000         self.assertEqual(stubs.capture_stdout.getvalue(),
1001                          stubs.expect_container_request_uuid + '\n')
1002         self.assertEqual(exited, 0)
1003
1004     @stubs
1005     def test_submit_missing_input(self, stubs):
1006         exited = arvados_cwl.main(
1007             ["--submit", "--no-wait", "--api=containers", "--debug",
1008              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1009             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1010         self.assertEqual(exited, 0)
1011
1012         exited = arvados_cwl.main(
1013             ["--submit", "--no-wait", "--api=containers", "--debug",
1014              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
1015             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1016         self.assertEqual(exited, 1)
1017
1018     @stubs
1019     def test_submit_container_project(self, stubs):
1020         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1021         exited = arvados_cwl.main(
1022             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1023                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1024             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1025
1026         expect_container = copy.deepcopy(stubs.expect_container_spec)
1027         expect_container["owner_uuid"] = project_uuid
1028         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1029                                        '--no-log-timestamps', '--disable-validate',
1030                                        "--eval-timeout=20", "--thread-count=1",
1031                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
1032                                        '--on-error=continue',
1033                                        '--project-uuid='+project_uuid,
1034                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1035
1036         stubs.api.container_requests().create.assert_called_with(
1037             body=JsonDiffMatcher(expect_container))
1038         self.assertEqual(stubs.capture_stdout.getvalue(),
1039                          stubs.expect_container_request_uuid + '\n')
1040         self.assertEqual(exited, 0)
1041
1042     @stubs
1043     def test_submit_container_eval_timeout(self, stubs):
1044         exited = arvados_cwl.main(
1045             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1046                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1047             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1048
1049         expect_container = copy.deepcopy(stubs.expect_container_spec)
1050         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1051                                        '--no-log-timestamps', '--disable-validate',
1052                                        '--eval-timeout=60.0', '--thread-count=1',
1053                                        '--enable-reuse', "--collection-cache-size=256",
1054                                        '--debug', '--on-error=continue',
1055                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1056
1057         stubs.api.container_requests().create.assert_called_with(
1058             body=JsonDiffMatcher(expect_container))
1059         self.assertEqual(stubs.capture_stdout.getvalue(),
1060                          stubs.expect_container_request_uuid + '\n')
1061         self.assertEqual(exited, 0)
1062
1063     @stubs
1064     def test_submit_container_collection_cache(self, stubs):
1065         exited = arvados_cwl.main(
1066             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
1067                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1068             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1069
1070         expect_container = copy.deepcopy(stubs.expect_container_spec)
1071         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1072                                        '--no-log-timestamps', '--disable-validate',
1073                                        '--eval-timeout=20', '--thread-count=1',
1074                                        '--enable-reuse', "--collection-cache-size=500",
1075                                        '--debug', '--on-error=continue',
1076                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1077         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1078
1079         stubs.api.container_requests().create.assert_called_with(
1080             body=JsonDiffMatcher(expect_container))
1081         self.assertEqual(stubs.capture_stdout.getvalue(),
1082                          stubs.expect_container_request_uuid + '\n')
1083         self.assertEqual(exited, 0)
1084
1085     @stubs
1086     def test_submit_container_thread_count(self, stubs):
1087         exited = arvados_cwl.main(
1088             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1089                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1090             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1091
1092         expect_container = copy.deepcopy(stubs.expect_container_spec)
1093         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1094                                        '--no-log-timestamps', '--disable-validate',
1095                                        '--eval-timeout=20', '--thread-count=20',
1096                                        '--enable-reuse', "--collection-cache-size=256",
1097                                        '--debug', '--on-error=continue',
1098                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1099
1100         stubs.api.container_requests().create.assert_called_with(
1101             body=JsonDiffMatcher(expect_container))
1102         self.assertEqual(stubs.capture_stdout.getvalue(),
1103                          stubs.expect_container_request_uuid + '\n')
1104         self.assertEqual(exited, 0)
1105
1106     @stubs
1107     def test_submit_job_runner_image(self, stubs):
1108         exited = arvados_cwl.main(
1109             ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1110                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1111             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1112
1113         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
1114
1115         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1116         stubs.api.pipeline_instances().create.assert_called_with(
1117             body=JsonDiffMatcher(expect_pipeline))
1118         self.assertEqual(stubs.capture_stdout.getvalue(),
1119                          stubs.expect_pipeline_uuid + '\n')
1120         self.assertEqual(exited, 0)
1121
1122     @stubs
1123     def test_submit_container_runner_image(self, stubs):
1124         exited = arvados_cwl.main(
1125             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1126                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1127             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1128
1129         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1130
1131         expect_container = copy.deepcopy(stubs.expect_container_spec)
1132         stubs.api.container_requests().create.assert_called_with(
1133             body=JsonDiffMatcher(expect_container))
1134         self.assertEqual(stubs.capture_stdout.getvalue(),
1135                          stubs.expect_container_request_uuid + '\n')
1136         self.assertEqual(exited, 0)
1137
1138     @stubs
1139     def test_submit_priority(self, stubs):
1140         exited = arvados_cwl.main(
1141             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1142                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1143             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1144
1145         stubs.expect_container_spec["priority"] = 669
1146
1147         expect_container = copy.deepcopy(stubs.expect_container_spec)
1148         stubs.api.container_requests().create.assert_called_with(
1149             body=JsonDiffMatcher(expect_container))
1150         self.assertEqual(stubs.capture_stdout.getvalue(),
1151                          stubs.expect_container_request_uuid + '\n')
1152         self.assertEqual(exited, 0)
1153
1154     @stubs
1155     def test_submit_wf_runner_resources(self, stubs):
1156         exited = arvados_cwl.main(
1157             ["--submit", "--no-wait", "--api=containers", "--debug",
1158                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1159             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1160
1161         expect_container = copy.deepcopy(stubs.expect_container_spec)
1162         expect_container["runtime_constraints"] = {
1163             "API": True,
1164             "vcpus": 2,
1165             "ram": (2000+512) * 2**20
1166         }
1167         expect_container["name"] = "submit_wf_runner_resources.cwl"
1168         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1169             {
1170                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1171                 "coresMin": 2,
1172                 "ramMin": 2000,
1173                 "keep_cache": 512
1174             }
1175         ]
1176         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1177             "arv": "http://arvados.org/cwl#",
1178         }
1179         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1180                         '--no-log-timestamps', '--disable-validate',
1181                         '--eval-timeout=20', '--thread-count=1',
1182                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1183                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1184
1185         stubs.api.container_requests().create.assert_called_with(
1186             body=JsonDiffMatcher(expect_container))
1187         self.assertEqual(stubs.capture_stdout.getvalue(),
1188                          stubs.expect_container_request_uuid + '\n')
1189         self.assertEqual(exited, 0)
1190
1191     def tearDown(self):
1192         arvados_cwl.arvdocker.arv_docker_clear_cache()
1193
1194     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1195     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1196     @mock.patch("arvados.api")
1197     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1198         arvados_cwl.arvdocker.arv_docker_clear_cache()
1199
1200         arvrunner = mock.MagicMock()
1201         arvrunner.project_uuid = ""
1202         api.return_value = mock.MagicMock()
1203         arvrunner.api = api.return_value
1204         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1205                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1206                                                                         "link_class": "docker_image_repo+tag",
1207                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1208                                                                         "owner_uuid": "",
1209                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1210                                                             {"items": [{"created_at": "",
1211                                                                         "head_uuid": "",
1212                                                                         "link_class": "docker_image_hash",
1213                                                                         "name": "123456",
1214                                                                         "owner_uuid": "",
1215                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1216         )
1217         find_one_image_hash.return_value = "123456"
1218
1219         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1220                                                                               "owner_uuid": "",
1221                                                                               "manifest_text": "",
1222                                                                               "properties": ""
1223                                                                           }], "items_available": 1, "offset": 0},)
1224         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1225         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1226                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1227         self.assertEqual("9999999999999999999999999999999b+99",
1228                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1229
1230
1231     @stubs
1232     def test_submit_secrets(self, stubs):
1233         exited = arvados_cwl.main(
1234             ["--submit", "--no-wait", "--api=containers", "--debug",
1235                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1236             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1237
1238         expect_container = {
1239             "command": [
1240                 "arvados-cwl-runner",
1241                 "--local",
1242                 "--api=containers",
1243                 "--no-log-timestamps",
1244                 "--disable-validate",
1245                 "--eval-timeout=20",
1246                 '--thread-count=1',
1247                 "--enable-reuse",
1248                 "--collection-cache-size=256",
1249                 '--debug',
1250                 "--on-error=continue",
1251                 "/var/lib/cwl/workflow.json#main",
1252                 "/var/lib/cwl/cwl.input.json"
1253             ],
1254             "container_image": "999999999999999999999999999999d3+99",
1255             "cwd": "/var/spool/cwl",
1256             "mounts": {
1257                 "/var/lib/cwl/cwl.input.json": {
1258                     "content": {
1259                         "pw": {
1260                             "$include": "/secrets/s0"
1261                         }
1262                     },
1263                     "kind": "json"
1264                 },
1265                 "/var/lib/cwl/workflow.json": {
1266                     "content": {
1267                         "$graph": [
1268                             {
1269                                 "$namespaces": {
1270                                     "cwltool": "http://commonwl.org/cwltool#"
1271                                 },
1272                                 "arguments": [
1273                                     "md5sum",
1274                                     "example.conf"
1275                                 ],
1276                                 "class": "CommandLineTool",
1277                                 "hints": [
1278                                     {
1279                                         "class": "http://commonwl.org/cwltool#Secrets",
1280                                         "secrets": [
1281                                             "#secret_job.cwl/pw"
1282                                         ]
1283                                     }
1284                                 ],
1285                                 "id": "#secret_job.cwl",
1286                                 "inputs": [
1287                                     {
1288                                         "id": "#secret_job.cwl/pw",
1289                                         "type": "string"
1290                                     }
1291                                 ],
1292                                 "outputs": [
1293                                     {
1294                                         "id": "#secret_job.cwl/out",
1295                                         "type": "stdout"
1296                                     }
1297                                 ],
1298                                 "stdout": "hashed_example.txt",
1299                                 "requirements": [
1300                                     {
1301                                         "class": "InitialWorkDirRequirement",
1302                                         "listing": [
1303                                             {
1304                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1305                                                 "entryname": "example.conf"
1306                                             }
1307                                         ]
1308                                     }
1309                                 ]
1310                             },
1311                             {
1312                                 "class": "Workflow",
1313                                 "hints": [
1314                                     {
1315                                         "class": "DockerRequirement",
1316                                         "dockerPull": "debian:8",
1317                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1318                                     },
1319                                     {
1320                                         "class": "http://commonwl.org/cwltool#Secrets",
1321                                         "secrets": [
1322                                             "#main/pw"
1323                                         ]
1324                                     }
1325                                 ],
1326                                 "id": "#main",
1327                                 "inputs": [
1328                                     {
1329                                         "id": "#main/pw",
1330                                         "type": "string"
1331                                     }
1332                                 ],
1333                                 "outputs": [
1334                                     {
1335                                         "id": "#main/out",
1336                                         "outputSource": "#main/step1/out",
1337                                         "type": "File"
1338                                     }
1339                                 ],
1340                                 "steps": [
1341                                     {
1342                                         "id": "#main/step1",
1343                                         "in": [
1344                                             {
1345                                                 "id": "#main/step1/pw",
1346                                                 "source": "#main/pw"
1347                                             }
1348                                         ],
1349                                         "out": [
1350                                             "#main/step1/out"
1351                                         ],
1352                                         "run": "#secret_job.cwl"
1353                                     }
1354                                 ]
1355                             }
1356                         ],
1357                         "cwlVersion": "v1.0"
1358                     },
1359                     "kind": "json"
1360                 },
1361                 "/var/spool/cwl": {
1362                     "kind": "collection",
1363                     "writable": True
1364                 },
1365                 "stdout": {
1366                     "kind": "file",
1367                     "path": "/var/spool/cwl/cwl.output.json"
1368                 }
1369             },
1370             "name": "secret_wf.cwl",
1371             "output_path": "/var/spool/cwl",
1372             "priority": 500,
1373             "properties": {},
1374             "runtime_constraints": {
1375                 "API": True,
1376                 "ram": 1342177280,
1377                 "vcpus": 1
1378             },
1379             "secret_mounts": {
1380                 "/secrets/s0": {
1381                     "content": "blorp",
1382                     "kind": "text"
1383                 }
1384             },
1385             "state": "Committed",
1386             "use_existing": True
1387         }
1388
1389         stubs.api.container_requests().create.assert_called_with(
1390             body=JsonDiffMatcher(expect_container))
1391         self.assertEqual(stubs.capture_stdout.getvalue(),
1392                          stubs.expect_container_request_uuid + '\n')
1393         self.assertEqual(exited, 0)
1394
1395     @stubs
1396     def test_submit_request_uuid(self, stubs):
1397         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1398         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1399
1400         stubs.api.container_requests().update().execute.return_value = {
1401             "uuid": stubs.expect_container_request_uuid,
1402             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1403             "state": "Queued"
1404         }
1405
1406         exited = arvados_cwl.main(
1407             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1408                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1410
1411         stubs.api.container_requests().update.assert_called_with(
1412             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1413         self.assertEqual(stubs.capture_stdout.getvalue(),
1414                          stubs.expect_container_request_uuid + '\n')
1415         self.assertEqual(exited, 0)
1416
1417     @stubs
1418     def test_submit_container_cluster_id(self, stubs):
1419         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1420
1421         exited = arvados_cwl.main(
1422             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1423                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1424             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1425
1426         expect_container = copy.deepcopy(stubs.expect_container_spec)
1427
1428         stubs.api.container_requests().create.assert_called_with(
1429             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1430         self.assertEqual(stubs.capture_stdout.getvalue(),
1431                          stubs.expect_container_request_uuid + '\n')
1432         self.assertEqual(exited, 0)
1433
1434     @stubs
1435     def test_submit_validate_cluster_id(self, stubs):
1436         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1437         exited = arvados_cwl.main(
1438             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1439              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1440             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1441         self.assertEqual(exited, 1)
1442
1443     @mock.patch("arvados.collection.CollectionReader")
1444     @stubs
1445     def test_submit_uuid_inputs(self, stubs, collectionReader):
1446         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1447         def list_side_effect(**kwargs):
1448             m = mock.MagicMock()
1449             if "count" in kwargs:
1450                 m.execute.return_value = {"items": [
1451                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1452                 ]}
1453             else:
1454                 m.execute.return_value = {"items": []}
1455             return m
1456         stubs.api.collections().list.side_effect = list_side_effect
1457
1458         exited = arvados_cwl.main(
1459             ["--submit", "--no-wait", "--api=containers", "--debug",
1460                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1461             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1462
1463         expect_container = copy.deepcopy(stubs.expect_container_spec)
1464         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1465         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1466         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1467
1468         stubs.api.collections().list.assert_has_calls([
1469             mock.call(count='none',
1470                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1471                       select=['uuid', 'portable_data_hash'])])
1472         stubs.api.container_requests().create.assert_called_with(
1473             body=JsonDiffMatcher(expect_container))
1474         self.assertEqual(stubs.capture_stdout.getvalue(),
1475                          stubs.expect_container_request_uuid + '\n')
1476         self.assertEqual(exited, 0)
1477
1478     @stubs
1479     def test_submit_mismatched_uuid_inputs(self, stubs):
1480         def list_side_effect(**kwargs):
1481             m = mock.MagicMock()
1482             if "count" in kwargs:
1483                 m.execute.return_value = {"items": [
1484                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1485                 ]}
1486             else:
1487                 m.execute.return_value = {"items": []}
1488             return m
1489         stubs.api.collections().list.side_effect = list_side_effect
1490
1491         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1492             capture_stderr = StringIO()
1493             cwltool_logger = logging.getLogger('cwltool')
1494             stderr_logger = logging.StreamHandler(capture_stderr)
1495             cwltool_logger.addHandler(stderr_logger)
1496
1497             try:
1498                 exited = arvados_cwl.main(
1499                     ["--submit", "--no-wait", "--api=containers", "--debug",
1500                         "tests/wf/submit_wf.cwl", infile],
1501                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1502
1503                 self.assertEqual(exited, 1)
1504                 self.assertRegexpMatches(
1505                     capture_stderr.getvalue(),
1506                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1507             finally:
1508                 cwltool_logger.removeHandler(stderr_logger)
1509
1510     @mock.patch("arvados.collection.CollectionReader")
1511     @stubs
1512     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1513         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1514         capture_stderr = StringIO()
1515
1516         cwltool_logger = logging.getLogger('cwltool')
1517         stderr_logger = logging.StreamHandler(capture_stderr)
1518         cwltool_logger.addHandler(stderr_logger)
1519
1520         exited = arvados_cwl.main(
1521             ["--submit", "--no-wait", "--api=containers", "--debug",
1522                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1523             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1524
1525         try:
1526             self.assertEqual(exited, 1)
1527             self.assertRegexpMatches(
1528                 capture_stderr.getvalue(),
1529                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1530         finally:
1531             cwltool_logger.removeHandler(stderr_logger)
1532
1533
1534 class TestCreateTemplate(unittest.TestCase):
1535     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1536
1537     def _adjust_script_params(self, expect_component):
1538         expect_component['script_parameters']['x'] = {
1539             'dataclass': 'File',
1540             'required': True,
1541             'type': 'File',
1542             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1543         }
1544         expect_component['script_parameters']['y'] = {
1545             'dataclass': 'Collection',
1546             'required': True,
1547             'type': 'Directory',
1548             'value': '99999999999999999999999999999998+99',
1549         }
1550         expect_component['script_parameters']['z'] = {
1551             'dataclass': 'Collection',
1552             'required': True,
1553             'type': 'Directory',
1554         }
1555
1556     @stubs
1557     def test_create(self, stubs):
1558         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1559
1560         exited = arvados_cwl.main(
1561             ["--create-workflow", "--debug",
1562              "--api=jobs",
1563              "--project-uuid", project_uuid,
1564              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1565             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1566
1567         stubs.api.pipeline_instances().create.refute_called()
1568         stubs.api.jobs().create.refute_called()
1569
1570         expect_component = copy.deepcopy(stubs.expect_job_spec)
1571         self._adjust_script_params(expect_component)
1572         expect_template = {
1573             "components": {
1574                 "submit_wf.cwl": expect_component,
1575             },
1576             "name": "submit_wf.cwl",
1577             "owner_uuid": project_uuid,
1578         }
1579         stubs.api.pipeline_templates().create.assert_called_with(
1580             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1581
1582         self.assertEqual(stubs.capture_stdout.getvalue(),
1583                          stubs.expect_pipeline_template_uuid + '\n')
1584         self.assertEqual(exited, 0)
1585
1586     @stubs
1587     def test_create_name(self, stubs):
1588         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1589
1590         exited = arvados_cwl.main(
1591             ["--create-workflow", "--debug",
1592              "--project-uuid", project_uuid,
1593              "--api=jobs",
1594              "--name", "testing 123",
1595              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1596             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1597
1598         stubs.api.pipeline_instances().create.refute_called()
1599         stubs.api.jobs().create.refute_called()
1600
1601         expect_component = copy.deepcopy(stubs.expect_job_spec)
1602         self._adjust_script_params(expect_component)
1603         expect_template = {
1604             "components": {
1605                 "testing 123": expect_component,
1606             },
1607             "name": "testing 123",
1608             "owner_uuid": project_uuid,
1609         }
1610         stubs.api.pipeline_templates().create.assert_called_with(
1611             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1612
1613         self.assertEqual(stubs.capture_stdout.getvalue(),
1614                          stubs.expect_pipeline_template_uuid + '\n')
1615         self.assertEqual(exited, 0)
1616
1617     @stubs
1618     def test_update_name(self, stubs):
1619         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1620
1621         exited = arvados_cwl.main(
1622             ["--update-workflow", self.existing_template_uuid,
1623              "--debug",
1624              "--project-uuid", project_uuid,
1625              "--api=jobs",
1626              "--name", "testing 123",
1627              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1628             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1629
1630         stubs.api.pipeline_instances().create.refute_called()
1631         stubs.api.jobs().create.refute_called()
1632
1633         expect_component = copy.deepcopy(stubs.expect_job_spec)
1634         self._adjust_script_params(expect_component)
1635         expect_template = {
1636             "components": {
1637                 "testing 123": expect_component,
1638             },
1639             "name": "testing 123",
1640             "owner_uuid": project_uuid,
1641         }
1642         stubs.api.pipeline_templates().create.refute_called()
1643         stubs.api.pipeline_templates().update.assert_called_with(
1644             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1645
1646         self.assertEqual(stubs.capture_stdout.getvalue(),
1647                          self.existing_template_uuid + '\n')
1648         self.assertEqual(exited, 0)
1649
1650
1651 class TestCreateWorkflow(unittest.TestCase):
1652     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1653     expect_workflow = StripYAMLComments(
1654         open("tests/wf/expect_packed.cwl").read())
1655
1656     @stubs
1657     def test_create(self, stubs):
1658         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1659
1660         exited = arvados_cwl.main(
1661             ["--create-workflow", "--debug",
1662              "--api=containers",
1663              "--project-uuid", project_uuid,
1664              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1665             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1666
1667         stubs.api.pipeline_templates().create.refute_called()
1668         stubs.api.container_requests().create.refute_called()
1669
1670         body = {
1671             "workflow": {
1672                 "owner_uuid": project_uuid,
1673                 "name": "submit_wf.cwl",
1674                 "description": "",
1675                 "definition": self.expect_workflow,
1676             }
1677         }
1678         stubs.api.workflows().create.assert_called_with(
1679             body=JsonDiffMatcher(body))
1680
1681         self.assertEqual(stubs.capture_stdout.getvalue(),
1682                          stubs.expect_workflow_uuid + '\n')
1683         self.assertEqual(exited, 0)
1684
1685     @stubs
1686     def test_create_name(self, stubs):
1687         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1688
1689         exited = arvados_cwl.main(
1690             ["--create-workflow", "--debug",
1691              "--api=containers",
1692              "--project-uuid", project_uuid,
1693              "--name", "testing 123",
1694              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1695             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1696
1697         stubs.api.pipeline_templates().create.refute_called()
1698         stubs.api.container_requests().create.refute_called()
1699
1700         body = {
1701             "workflow": {
1702                 "owner_uuid": project_uuid,
1703                 "name": "testing 123",
1704                 "description": "",
1705                 "definition": self.expect_workflow,
1706             }
1707         }
1708         stubs.api.workflows().create.assert_called_with(
1709             body=JsonDiffMatcher(body))
1710
1711         self.assertEqual(stubs.capture_stdout.getvalue(),
1712                          stubs.expect_workflow_uuid + '\n')
1713         self.assertEqual(exited, 0)
1714
1715     @stubs
1716     def test_incompatible_api(self, stubs):
1717         capture_stderr = StringIO()
1718         acr_logger = logging.getLogger('arvados.cwl-runner')
1719         stderr_logger = logging.StreamHandler(capture_stderr)
1720         acr_logger.addHandler(stderr_logger)
1721
1722         try:
1723             exited = arvados_cwl.main(
1724                 ["--update-workflow", self.existing_workflow_uuid,
1725                  "--api=jobs",
1726                  "--debug",
1727                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1728                 sys.stderr, sys.stderr, api_client=stubs.api)
1729             self.assertEqual(exited, 1)
1730             self.assertRegexpMatches(
1731                 capture_stderr.getvalue(),
1732                 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1733         finally:
1734             acr_logger.removeHandler(stderr_logger)
1735
1736     @stubs
1737     def test_update(self, stubs):
1738         exited = arvados_cwl.main(
1739             ["--update-workflow", self.existing_workflow_uuid,
1740              "--debug",
1741              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1742             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1743
1744         body = {
1745             "workflow": {
1746                 "name": "submit_wf.cwl",
1747                 "description": "",
1748                 "definition": self.expect_workflow,
1749             }
1750         }
1751         stubs.api.workflows().update.assert_called_with(
1752             uuid=self.existing_workflow_uuid,
1753             body=JsonDiffMatcher(body))
1754         self.assertEqual(stubs.capture_stdout.getvalue(),
1755                          self.existing_workflow_uuid + '\n')
1756         self.assertEqual(exited, 0)
1757
1758     @stubs
1759     def test_update_name(self, stubs):
1760         exited = arvados_cwl.main(
1761             ["--update-workflow", self.existing_workflow_uuid,
1762              "--debug", "--name", "testing 123",
1763              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1764             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1765
1766         body = {
1767             "workflow": {
1768                 "name": "testing 123",
1769                 "description": "",
1770                 "definition": self.expect_workflow,
1771             }
1772         }
1773         stubs.api.workflows().update.assert_called_with(
1774             uuid=self.existing_workflow_uuid,
1775             body=JsonDiffMatcher(body))
1776         self.assertEqual(stubs.capture_stdout.getvalue(),
1777                          self.existing_workflow_uuid + '\n')
1778         self.assertEqual(exited, 0)
1779
1780     @stubs
1781     def test_create_collection_per_tool(self, stubs):
1782         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1783
1784         exited = arvados_cwl.main(
1785             ["--create-workflow", "--debug",
1786              "--api=containers",
1787              "--project-uuid", project_uuid,
1788              "tests/collection_per_tool/collection_per_tool.cwl"],
1789             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1790
1791         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1792         expect_workflow = StripYAMLComments(open(toolfile).read())
1793
1794         body = {
1795             "workflow": {
1796                 "owner_uuid": project_uuid,
1797                 "name": "collection_per_tool.cwl",
1798                 "description": "",
1799                 "definition": expect_workflow,
1800             }
1801         }
1802         stubs.api.workflows().create.assert_called_with(
1803             body=JsonDiffMatcher(body))
1804
1805         self.assertEqual(stubs.capture_stdout.getvalue(),
1806                          stubs.expect_workflow_uuid + '\n')
1807         self.assertEqual(exited, 0)
1808
1809 class TestTemplateInputs(unittest.TestCase):
1810     expect_template = {
1811         "components": {
1812             "inputs_test.cwl": {
1813                 'runtime_constraints': {
1814                     'docker_image': '999999999999999999999999999999d3+99',
1815                     'min_ram_mb_per_node': 1024
1816                 },
1817                 'script_parameters': {
1818                     'cwl:tool':
1819                     'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
1820                     'optionalFloatInput': None,
1821                     'fileInput': {
1822                         'type': 'File',
1823                         'dataclass': 'File',
1824                         'required': True,
1825                         'title': "It's a file; we expect to find some characters in it.",
1826                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1827                     },
1828                     'floatInput': {
1829                         'type': 'float',
1830                         'dataclass': 'number',
1831                         'required': True,
1832                         'title': 'Floats like a duck',
1833                         'default': 0.1,
1834                         'value': 0.1,
1835                     },
1836                     'optionalFloatInput': {
1837                         'type': ['null', 'float'],
1838                         'dataclass': 'number',
1839                         'required': False,
1840                     },
1841                     'boolInput': {
1842                         'type': 'boolean',
1843                         'dataclass': 'boolean',
1844                         'required': True,
1845                         'title': 'True or false?',
1846                     },
1847                 },
1848                 'repository': 'arvados',
1849                 'script_version': 'master',
1850                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1851                 'script': 'cwl-runner',
1852             },
1853         },
1854         "name": "inputs_test.cwl",
1855     }
1856
1857     @stubs
1858     def test_inputs_empty(self, stubs):
1859         exited = arvados_cwl.main(
1860             ["--create-template",
1861              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1862             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1863
1864         stubs.api.pipeline_templates().create.assert_called_with(
1865             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1866
1867         self.assertEqual(exited, 0)
1868
1869     @stubs
1870     def test_inputs(self, stubs):
1871         exited = arvados_cwl.main(
1872             ["--create-template",
1873              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1874             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1875
1876         expect_template = copy.deepcopy(self.expect_template)
1877         params = expect_template[
1878             "components"]["inputs_test.cwl"]["script_parameters"]
1879         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1880         params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
1881         params["floatInput"]["value"] = 1.234
1882         params["boolInput"]["value"] = True
1883
1884         stubs.api.pipeline_templates().create.assert_called_with(
1885             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1886         self.assertEqual(exited, 0)