sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21
  22 from io import BytesIO
  23
  24 # StringIO.StringIO and io.StringIO have different behavior write() is
  25 # called with both python2 (byte) strings and unicode strings
  26 # (specifically there's some logging in cwltool that causes trouble).
  27 # This isn't a problem on python3 because all string are unicode.
  28 if sys.version_info[0] < 3:
  29     from StringIO import StringIO
  30 else:
  31     from io import StringIO
  32
  33 import arvados
  34 import arvados.collection
  35 import arvados_cwl
  36 import arvados_cwl.executor
  37 import arvados_cwl.runner
  38 import arvados.keep
  39
  40 from .matcher import JsonDiffMatcher, StripYAMLComments
  41 from .mock_discovery import get_rootDesc
  42
  43 import ruamel.yaml as yaml
  44
  45 _rootDesc = None
  46
  47 def stubs(func):
  48     @functools.wraps(func)
  49     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  50     @mock.patch("arvados.collection.KeepClient")
  51     @mock.patch("arvados.keep.KeepClient")
  52     @mock.patch("arvados.events.subscribe")
  53     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  54         class Stubs(object):
  55             pass
  56         stubs = Stubs()
  57         stubs.events = events
  58         stubs.keepdocker = keepdocker
  59
  60         def putstub(p, **kwargs):
  61             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  62         keep_client1().put.side_effect = putstub
  63         keep_client1.put.side_effect = putstub
  64         keep_client2().put.side_effect = putstub
  65         keep_client2.put.side_effect = putstub
  66
  67         stubs.keep_client = keep_client2
  68         stubs.docker_images = {
  69             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  70             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  71             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  72             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  73         }
  74         def kd(a, b, image_name=None, image_tag=None):
  75             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  76         stubs.keepdocker.side_effect = kd
  77
  78         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  79         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  80
  81         if sys.version_info[0] < 3:
  82             stubs.capture_stdout = BytesIO()
  83         else:
  84             stubs.capture_stdout = StringIO()
  85
  86         stubs.api = mock.MagicMock()
  87         stubs.api._rootDesc = get_rootDesc()
  88         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  89
  90         stubs.api.users().current().execute.return_value = {
  91             "uuid": stubs.fake_user_uuid,
  92         }
  93         stubs.api.collections().list().execute.return_value = {"items": []}
  94         stubs.api.containers().current().execute.return_value = {
  95             "uuid": stubs.fake_container_uuid,
  96         }
  97
  98         class CollectionExecute(object):
  99             def __init__(self, exe):
 100                 self.exe = exe
 101             def execute(self, num_retries=None):
 102                 return self.exe
 103
 104         def collection_createstub(created_collections, body, ensure_unique_name=None):
 105             mt = body["manifest_text"].encode('utf-8')
 106             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 107             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 108             created_collections[uuid] = {
 109                 "uuid": uuid,
 110                 "portable_data_hash": pdh,
 111                 "manifest_text": mt.decode('utf-8')
 112             }
 113             return CollectionExecute(created_collections[uuid])
 114
 115         def collection_getstub(created_collections, uuid):
 116             for v in viewvalues(created_collections):
 117                 if uuid in (v["uuid"], v["portable_data_hash"]):
 118                     return CollectionExecute(v)
 119
 120         created_collections = {
 121             "99999999999999999999999999999998+99": {
 122                 "uuid": "",
 123                 "portable_data_hash": "99999999999999999999999999999998+99",
 124                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 125             },
 126             "99999999999999999999999999999997+99": {
 127                 "uuid": "",
 128                 "portable_data_hash": "99999999999999999999999999999997+99",
 129                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 130             },
 131             "99999999999999999999999999999994+99": {
 132                 "uuid": "",
 133                 "portable_data_hash": "99999999999999999999999999999994+99",
 134                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 135             },
 136             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 137                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 138                 "portable_data_hash": "999999999999999999999999999999d3+99",
 139                 "manifest_text": ""
 140             },
 141             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 142                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 143                 "portable_data_hash": "999999999999999999999999999999d4+99",
 144                 "manifest_text": ""
 145             },
 146             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 147                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 148                 "portable_data_hash": "999999999999999999999999999999d5+99",
 149                 "manifest_text": ""
 150             },
 151             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 152                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 153                 "portable_data_hash": "999999999999999999999999999999d6+99",
 154                 "manifest_text": ""
 155             }
 156         }
 157         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 158         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 159
 160         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 161         stubs.api.jobs().create().execute.return_value = {
 162             "uuid": stubs.expect_job_uuid,
 163             "state": "Queued",
 164         }
 165
 166         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 167         stubs.api.container_requests().create().execute.return_value = {
 168             "uuid": stubs.expect_container_request_uuid,
 169             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 170             "state": "Queued"
 171         }
 172
 173         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 174         stubs.api.pipeline_templates().create().execute.return_value = {
 175             "uuid": stubs.expect_pipeline_template_uuid,
 176         }
 177         stubs.expect_job_spec = {
 178             'runtime_constraints': {
 179                 'docker_image': '999999999999999999999999999999d3+99',
 180                 'min_ram_mb_per_node': 1024
 181             },
 182             'script_parameters': {
 183                 'x': {
 184                     'basename': 'blorp.txt',
 185                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 186                     'class': 'File'
 187                 },
 188                 'y': {
 189                     'basename': '99999999999999999999999999999998+99',
 190                     'location': 'keep:99999999999999999999999999999998+99',
 191                     'class': 'Directory'
 192                 },
 193                 'z': {
 194                     'basename': 'anonymous',
 195                     "listing": [{
 196                         "basename": "renamed.txt",
 197                         "class": "File",
 198                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 199                         "size": 0
 200                     }],
 201                     'class': 'Directory'
 202                 },
 203                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 204             },
 205             'repository': 'arvados',
 206             'script_version': 'master',
 207             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 208             'script': 'cwl-runner'
 209         }
 210         stubs.pipeline_component = stubs.expect_job_spec.copy()
 211         stubs.expect_pipeline_instance = {
 212             'name': 'submit_wf.cwl',
 213             'state': 'RunningOnServer',
 214             'owner_uuid': None,
 215             "components": {
 216                 "cwl-runner": {
 217                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 218                     'script_parameters': {
 219                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 220                         'x': {"value": {
 221                             'basename': 'blorp.txt',
 222                             'class': 'File',
 223                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 224                             "size": 16
 225                         }},
 226                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 227                               'listing': [
 228                                   {
 229                                       'basename': 'renamed.txt',
 230                                       'class': 'File', 'location':
 231                                       'keep:99999999999999999999999999999998+99/file1.txt',
 232                                       'size': 0
 233                                   }
 234                               ]}},
 235                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 236                         'arv:debug': True,
 237                         'arv:enable_reuse': True,
 238                         'arv:on_error': 'continue'
 239                     },
 240                     'repository': 'arvados',
 241                     'script_version': 'master',
 242                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 243                     'script': 'cwl-runner',
 244                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 245                 }
 246             }
 247         }
 248         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 249         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 250         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 251         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 252         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 253             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 254             "state": "Queued"
 255         }
 256         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 257         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 258
 259         with open("tests/wf/submit_wf_packed.cwl") as f:
 260             expect_packed_workflow = yaml.round_trip_load(f)
 261
 262         stubs.expect_container_spec = {
 263             'priority': 500,
 264             'mounts': {
 265                 '/var/spool/cwl': {
 266                     'writable': True,
 267                     'kind': 'collection'
 268                 },
 269                 '/var/lib/cwl/workflow.json': {
 270                     'content': expect_packed_workflow,
 271                     'kind': 'json'
 272                 },
 273                 'stdout': {
 274                     'path': '/var/spool/cwl/cwl.output.json',
 275                     'kind': 'file'
 276                 },
 277                 '/var/lib/cwl/cwl.input.json': {
 278                     'kind': 'json',
 279                     'content': {
 280                         'y': {
 281                             'basename': '99999999999999999999999999999998+99',
 282                             'location': 'keep:99999999999999999999999999999998+99',
 283                             'class': 'Directory'},
 284                         'x': {
 285                             'basename': u'blorp.txt',
 286                             'class': 'File',
 287                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 288                             "size": 16
 289                         },
 290                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 291                             {'basename': 'renamed.txt',
 292                              'class': 'File',
 293                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 294                              'size': 0
 295                             }
 296                         ]}
 297                     },
 298                     'kind': 'json'
 299                 }
 300             },
 301             'secret_mounts': {},
 302             'state': 'Committed',
 303             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 304                         '--no-log-timestamps', '--disable-validate',
 305                         '--eval-timeout=20', '--thread-count=1',
 306                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 307                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 308             'name': 'submit_wf.cwl',
 309             'container_image': '999999999999999999999999999999d3+99',
 310             'output_path': '/var/spool/cwl',
 311             'cwd': '/var/spool/cwl',
 312             'runtime_constraints': {
 313                 'API': True,
 314                 'vcpus': 1,
 315                 'ram': (1024+256)*1024*1024
 316             },
 317             'use_existing': True,
 318             'properties': {},
 319             'secret_mounts': {}
 320         }
 321
 322         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 323         stubs.api.workflows().create().execute.return_value = {
 324             "uuid": stubs.expect_workflow_uuid,
 325         }
 326         def update_mock(**kwargs):
 327             stubs.updated_uuid = kwargs.get('uuid')
 328             return mock.DEFAULT
 329         stubs.api.workflows().update.side_effect = update_mock
 330         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 331             "uuid": stubs.updated_uuid,
 332         }
 333
 334         return func(self, stubs, *args, **kwargs)
 335     return wrapped
 336
 337
 338 class TestSubmit(unittest.TestCase):
 339
 340     def setUp(self):
 341         cwltool.process._names = set()
 342
 343     @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
 344     @mock.patch("time.sleep")
 345     @stubs
 346     def test_submit(self, stubs, tm, arvdock):
 347         def get_image(api_client, dockerRequirement, pull_image, project_uuid):
 348             if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
 349                 return '999999999999999999999999999999d3+99'
 350             elif dockerRequirement["dockerPull"] == "debian:8":
 351                 return '999999999999999999999999999999d4+99'
 352         arvdock.side_effect = get_image
 353
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--api=jobs", "--debug",
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 358
 359         stubs.api.collections().create.assert_has_calls([
 360             mock.call(body=JsonDiffMatcher({
 361                 'manifest_text':
 362                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 363                 'replication_desired': None,
 364                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 365             }), ensure_unique_name=False),
 366             mock.call(body=JsonDiffMatcher({
 367                 'manifest_text':
 368                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 369                 'replication_desired': None,
 370                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 371             }), ensure_unique_name=False),
 372             mock.call(body=JsonDiffMatcher({
 373                 'manifest_text':
 374                 ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
 375                 'replication_desired': None,
 376                 'name': 'submit_wf.cwl',
 377             }), ensure_unique_name=True)        ])
 378
 379         arvdock.assert_has_calls([
 380             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 381             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
 382             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 383         ])
 384
 385         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 386         stubs.api.pipeline_instances().create.assert_called_with(
 387             body=JsonDiffMatcher(expect_pipeline))
 388         self.assertEqual(stubs.capture_stdout.getvalue(),
 389                          stubs.expect_pipeline_uuid + '\n')
 390         self.assertEqual(exited, 0)
 391
 392     @mock.patch("time.sleep")
 393     @stubs
 394     def test_submit_no_reuse(self, stubs, tm):
 395         exited = arvados_cwl.main(
 396             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 397              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 398             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 399
 400         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 401         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 402         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 403
 404         stubs.api.pipeline_instances().create.assert_called_with(
 405             body=JsonDiffMatcher(expect_pipeline))
 406         self.assertEqual(stubs.capture_stdout.getvalue(),
 407                          stubs.expect_pipeline_uuid + '\n')
 408         self.assertEqual(exited, 0)
 409
 410     @stubs
 411     def test_error_when_multiple_storage_classes_specified(self, stubs):
 412         storage_classes = "foo,bar"
 413         exited = arvados_cwl.main(
 414                 ["--debug", "--storage-classes", storage_classes,
 415                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 416                 sys.stdin, sys.stderr, api_client=stubs.api)
 417         self.assertEqual(exited, 1)
 418
 419     @mock.patch("time.sleep")
 420     @stubs
 421     def test_submit_on_error(self, stubs, tm):
 422         exited = arvados_cwl.main(
 423             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 424              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 426
 427         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 428         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 429
 430         stubs.api.pipeline_instances().create.assert_called_with(
 431             body=JsonDiffMatcher(expect_pipeline))
 432         self.assertEqual(stubs.capture_stdout.getvalue(),
 433                          stubs.expect_pipeline_uuid + '\n')
 434         self.assertEqual(exited, 0)
 435
 436     @mock.patch("time.sleep")
 437     @stubs
 438     def test_submit_runner_ram(self, stubs, tm):
 439         exited = arvados_cwl.main(
 440             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 441              "--api=jobs",
 442              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 443             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 444
 445         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 446         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 447
 448         stubs.api.pipeline_instances().create.assert_called_with(
 449             body=JsonDiffMatcher(expect_pipeline))
 450         self.assertEqual(stubs.capture_stdout.getvalue(),
 451                          stubs.expect_pipeline_uuid + '\n')
 452         self.assertEqual(exited, 0)
 453
 454     @mock.patch("time.sleep")
 455     @stubs
 456     def test_submit_invalid_runner_ram(self, stubs, tm):
 457         exited = arvados_cwl.main(
 458             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 459              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 460             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 461         self.assertEqual(exited, 1)
 462
 463     @mock.patch("time.sleep")
 464     @stubs
 465     def test_submit_output_name(self, stubs, tm):
 466         output_name = "test_output_name"
 467
 468         exited = arvados_cwl.main(
 469             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 470              "--api=jobs",
 471              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 472             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 473
 474         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 475         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 476
 477         stubs.api.pipeline_instances().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_pipeline))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_pipeline_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @mock.patch("time.sleep")
 484     @stubs
 485     def test_submit_pipeline_name(self, stubs, tm):
 486         exited = arvados_cwl.main(
 487             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 488              "--api=jobs",
 489              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 490             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 491         self.assertEqual(exited, 0)
 492
 493         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 494         expect_pipeline["name"] = "hello job 123"
 495
 496         stubs.api.pipeline_instances().create.assert_called_with(
 497             body=JsonDiffMatcher(expect_pipeline))
 498         self.assertEqual(stubs.capture_stdout.getvalue(),
 499                          stubs.expect_pipeline_uuid + '\n')
 500
 501     @mock.patch("time.sleep")
 502     @stubs
 503     def test_submit_output_tags(self, stubs, tm):
 504         output_tags = "tag0,tag1,tag2"
 505
 506         exited = arvados_cwl.main(
 507             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 508              "--api=jobs",
 509              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 510             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 511         self.assertEqual(exited, 0)
 512
 513         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 514         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 515
 516         stubs.api.pipeline_instances().create.assert_called_with(
 517             body=JsonDiffMatcher(expect_pipeline))
 518         self.assertEqual(stubs.capture_stdout.getvalue(),
 519                          stubs.expect_pipeline_uuid + '\n')
 520
 521     @mock.patch("time.sleep")
 522     @stubs
 523     def test_submit_with_project_uuid(self, stubs, tm):
 524         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 525
 526         exited = arvados_cwl.main(
 527             ["--submit", "--no-wait", "--debug",
 528              "--project-uuid", project_uuid,
 529              "--api=jobs",
 530              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 531             sys.stdout, sys.stderr, api_client=stubs.api)
 532         self.assertEqual(exited, 0)
 533
 534         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 535         expect_pipeline["owner_uuid"] = project_uuid
 536         stubs.api.pipeline_instances().create.assert_called_with(
 537             body=JsonDiffMatcher(expect_pipeline))
 538
 539     @stubs
 540     def test_submit_container(self, stubs):
 541         exited = arvados_cwl.main(
 542             ["--submit", "--no-wait", "--api=containers", "--debug",
 543                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 544             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 545
 546         stubs.api.collections().create.assert_has_calls([
 547             mock.call(body=JsonDiffMatcher({
 548                 'manifest_text':
 549                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 550                 'replication_desired': None,
 551                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 552             }), ensure_unique_name=False),
 553             mock.call(body=JsonDiffMatcher({
 554                 'manifest_text':
 555                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 556                 'replication_desired': None,
 557                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 558             }), ensure_unique_name=False),
 559             ])
 560
 561         expect_container = copy.deepcopy(stubs.expect_container_spec)
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(stubs.capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566         self.assertEqual(exited, 0)
 567
 568     @stubs
 569     def test_submit_container_no_reuse(self, stubs):
 570         exited = arvados_cwl.main(
 571             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 572                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 573             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 574
 575         expect_container = copy.deepcopy(stubs.expect_container_spec)
 576         expect_container["command"] = [
 577             'arvados-cwl-runner', '--local', '--api=containers',
 578             '--no-log-timestamps', '--disable-validate',
 579             '--eval-timeout=20', '--thread-count=1',
 580             '--disable-reuse', "--collection-cache-size=256",
 581             '--debug', '--on-error=continue',
 582             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 583         expect_container["use_existing"] = False
 584
 585         stubs.api.container_requests().create.assert_called_with(
 586             body=JsonDiffMatcher(expect_container))
 587         self.assertEqual(stubs.capture_stdout.getvalue(),
 588                          stubs.expect_container_request_uuid + '\n')
 589         self.assertEqual(exited, 0)
 590
 591     @stubs
 592     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 593         exited = arvados_cwl.main(
 594             ["--submit", "--no-wait", "--api=containers", "--debug",
 595              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 596             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 597         self.assertEqual(exited, 0)
 598
 599         expect_container = copy.deepcopy(stubs.expect_container_spec)
 600         expect_container["command"] = [
 601             'arvados-cwl-runner', '--local', '--api=containers',
 602             '--no-log-timestamps', '--disable-validate',
 603             '--eval-timeout=20', '--thread-count=1',
 604             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 605             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 606         expect_container["use_existing"] = False
 607         expect_container["name"] = "submit_wf_no_reuse.cwl"
 608         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 609             {
 610                 "class": "http://arvados.org/cwl#ReuseRequirement",
 611                 "enableReuse": False,
 612             },
 613         ]
 614         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 615             "arv": "http://arvados.org/cwl#",
 616             "cwltool": "http://commonwl.org/cwltool#"
 617         }
 618
 619         stubs.api.container_requests().create.assert_called_with(
 620             body=JsonDiffMatcher(expect_container))
 621         self.assertEqual(stubs.capture_stdout.getvalue(),
 622                          stubs.expect_container_request_uuid + '\n')
 623
 624
 625     @stubs
 626     def test_submit_container_on_error(self, stubs):
 627         exited = arvados_cwl.main(
 628             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 629                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 630             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 631
 632         expect_container = copy.deepcopy(stubs.expect_container_spec)
 633         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 634                                        '--no-log-timestamps', '--disable-validate',
 635                                        '--eval-timeout=20', '--thread-count=1',
 636                                        '--enable-reuse', "--collection-cache-size=256",
 637                                        '--debug', '--on-error=stop',
 638                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 639
 640         stubs.api.container_requests().create.assert_called_with(
 641             body=JsonDiffMatcher(expect_container))
 642         self.assertEqual(stubs.capture_stdout.getvalue(),
 643                          stubs.expect_container_request_uuid + '\n')
 644         self.assertEqual(exited, 0)
 645
 646     @stubs
 647     def test_submit_container_output_name(self, stubs):
 648         output_name = "test_output_name"
 649
 650         exited = arvados_cwl.main(
 651             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 652                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 653             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 654
 655         expect_container = copy.deepcopy(stubs.expect_container_spec)
 656         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 657                                        '--no-log-timestamps', '--disable-validate',
 658                                        '--eval-timeout=20', '--thread-count=1',
 659                                        '--enable-reuse', "--collection-cache-size=256",
 660                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 661                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 662         expect_container["output_name"] = output_name
 663
 664         stubs.api.container_requests().create.assert_called_with(
 665             body=JsonDiffMatcher(expect_container))
 666         self.assertEqual(stubs.capture_stdout.getvalue(),
 667                          stubs.expect_container_request_uuid + '\n')
 668         self.assertEqual(exited, 0)
 669
 670     @stubs
 671     def test_submit_storage_classes(self, stubs):
 672         exited = arvados_cwl.main(
 673             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 674                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 676
 677         expect_container = copy.deepcopy(stubs.expect_container_spec)
 678         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 679                                        '--no-log-timestamps', '--disable-validate',
 680                                        '--eval-timeout=20', '--thread-count=1',
 681                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 682                                        "--storage-classes=foo", '--on-error=continue',
 683                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 684
 685         stubs.api.container_requests().create.assert_called_with(
 686             body=JsonDiffMatcher(expect_container))
 687         self.assertEqual(stubs.capture_stdout.getvalue(),
 688                          stubs.expect_container_request_uuid + '\n')
 689         self.assertEqual(exited, 0)
 690
 691     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 692     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 693     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 694     @stubs
 695     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 696         def set_final_output(job_order, output_callback, runtimeContext):
 697             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 698             return []
 699         job.side_effect = set_final_output
 700
 701         exited = arvados_cwl.main(
 702             ["--debug", "--local", "--storage-classes=foo",
 703                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 704             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 705
 706         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 707         self.assertEqual(exited, 0)
 708
 709     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 710     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 711     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 712     @stubs
 713     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 714         def set_final_output(job_order, output_callback, runtimeContext):
 715             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 716             return []
 717         job.side_effect = set_final_output
 718
 719         exited = arvados_cwl.main(
 720             ["--debug", "--local",
 721                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 722             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 723
 724         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 725         self.assertEqual(exited, 0)
 726
 727     @stubs
 728     def test_submit_container_output_ttl(self, stubs):
 729         exited = arvados_cwl.main(
 730             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 731                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 732             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 733
 734         expect_container = copy.deepcopy(stubs.expect_container_spec)
 735         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 736                                        '--no-log-timestamps', '--disable-validate',
 737                                        '--eval-timeout=20', '--thread-count=1',
 738                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 739                                        '--on-error=continue',
 740                                        "--intermediate-output-ttl=3600",
 741                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 742
 743         stubs.api.container_requests().create.assert_called_with(
 744             body=JsonDiffMatcher(expect_container))
 745         self.assertEqual(stubs.capture_stdout.getvalue(),
 746                          stubs.expect_container_request_uuid + '\n')
 747         self.assertEqual(exited, 0)
 748
 749     @stubs
 750     def test_submit_container_trash_intermediate(self, stubs):
 751         exited = arvados_cwl.main(
 752             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 753                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 754             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 755
 756
 757         expect_container = copy.deepcopy(stubs.expect_container_spec)
 758         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 759                                        '--no-log-timestamps', '--disable-validate',
 760                                        '--eval-timeout=20', '--thread-count=1',
 761                                        '--enable-reuse', "--collection-cache-size=256",
 762                                        '--debug', '--on-error=continue',
 763                                        "--trash-intermediate",
 764                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 765
 766         stubs.api.container_requests().create.assert_called_with(
 767             body=JsonDiffMatcher(expect_container))
 768         self.assertEqual(stubs.capture_stdout.getvalue(),
 769                          stubs.expect_container_request_uuid + '\n')
 770         self.assertEqual(exited, 0)
 771
 772     @stubs
 773     def test_submit_container_output_tags(self, stubs):
 774         output_tags = "tag0,tag1,tag2"
 775
 776         exited = arvados_cwl.main(
 777             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 778                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 779             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 780
 781         expect_container = copy.deepcopy(stubs.expect_container_spec)
 782         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 783                                        '--no-log-timestamps', '--disable-validate',
 784                                        '--eval-timeout=20', '--thread-count=1',
 785                                        '--enable-reuse', "--collection-cache-size=256",
 786                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 787                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 788
 789         stubs.api.container_requests().create.assert_called_with(
 790             body=JsonDiffMatcher(expect_container))
 791         self.assertEqual(stubs.capture_stdout.getvalue(),
 792                          stubs.expect_container_request_uuid + '\n')
 793         self.assertEqual(exited, 0)
 794
 795     @stubs
 796     def test_submit_container_runner_ram(self, stubs):
 797         exited = arvados_cwl.main(
 798             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 799                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 800             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 801
 802         expect_container = copy.deepcopy(stubs.expect_container_spec)
 803         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 804
 805         stubs.api.container_requests().create.assert_called_with(
 806             body=JsonDiffMatcher(expect_container))
 807         self.assertEqual(stubs.capture_stdout.getvalue(),
 808                          stubs.expect_container_request_uuid + '\n')
 809         self.assertEqual(exited, 0)
 810
 811     @mock.patch("arvados.collection.CollectionReader")
 812     @mock.patch("time.sleep")
 813     @stubs
 814     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 815         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 816         exited = arvados_cwl.main(
 817             ["--submit", "--no-wait", "--api=containers", "--debug",
 818              "tests/wf/submit_keepref_wf.cwl"],
 819             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 820         self.assertEqual(exited, 0)
 821
 822     @mock.patch("arvados.collection.CollectionReader")
 823     @mock.patch("time.sleep")
 824     @stubs
 825     def test_submit_keepref(self, stubs, tm, reader):
 826         with open("tests/wf/expect_arvworkflow.cwl") as f:
 827             reader().open().__enter__().read.return_value = f.read()
 828
 829         exited = arvados_cwl.main(
 830             ["--submit", "--no-wait", "--api=containers", "--debug",
 831              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 832             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 833
 834         expect_container = {
 835             'priority': 500,
 836             'mounts': {
 837                 '/var/spool/cwl': {
 838                     'writable': True,
 839                     'kind': 'collection'
 840                 },
 841                 'stdout': {
 842                     'path': '/var/spool/cwl/cwl.output.json',
 843                     'kind': 'file'
 844                 },
 845                 '/var/lib/cwl/workflow': {
 846                     'portable_data_hash': '99999999999999999999999999999994+99',
 847                     'kind': 'collection'
 848                 },
 849                 '/var/lib/cwl/cwl.input.json': {
 850                     'content': {
 851                         'x': 'XxX'
 852                     },
 853                     'kind': 'json'
 854                 }
 855             }, 'state': 'Committed',
 856             'output_path': '/var/spool/cwl',
 857             'name': 'expect_arvworkflow.cwl#main',
 858             'container_image': '999999999999999999999999999999d3+99',
 859             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 860                         '--no-log-timestamps', '--disable-validate',
 861                         '--eval-timeout=20', '--thread-count=1',
 862                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 863                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 864             'cwd': '/var/spool/cwl',
 865             'runtime_constraints': {
 866                 'API': True,
 867                 'vcpus': 1,
 868                 'ram': 1342177280
 869             },
 870             'use_existing': True,
 871             'properties': {},
 872             'secret_mounts': {}
 873         }
 874
 875         stubs.api.container_requests().create.assert_called_with(
 876             body=JsonDiffMatcher(expect_container))
 877         self.assertEqual(stubs.capture_stdout.getvalue(),
 878                          stubs.expect_container_request_uuid + '\n')
 879         self.assertEqual(exited, 0)
 880
 881     @mock.patch("arvados.collection.CollectionReader")
 882     @mock.patch("time.sleep")
 883     @stubs
 884     def test_submit_jobs_keepref(self, stubs, tm, reader):
 885         with open("tests/wf/expect_arvworkflow.cwl") as f:
 886             reader().open().__enter__().read.return_value = f.read()
 887
 888         exited = arvados_cwl.main(
 889             ["--submit", "--no-wait", "--api=jobs", "--debug",
 890              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 891             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 892
 893         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 894         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 895         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 896         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 897         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 898         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 899         stubs.api.pipeline_instances().create.assert_called_with(
 900             body=JsonDiffMatcher(expect_pipeline))
 901         self.assertEqual(exited, 0)
 902
 903     @mock.patch("time.sleep")
 904     @stubs
 905     def test_submit_arvworkflow(self, stubs, tm):
 906         with open("tests/wf/expect_arvworkflow.cwl") as f:
 907             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 908
 909         exited = arvados_cwl.main(
 910             ["--submit", "--no-wait", "--api=containers", "--debug",
 911              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 912             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 913
 914         expect_container = {
 915             'priority': 500,
 916             'mounts': {
 917                 '/var/spool/cwl': {
 918                     'writable': True,
 919                     'kind': 'collection'
 920                 },
 921                 'stdout': {
 922                     'path': '/var/spool/cwl/cwl.output.json',
 923                     'kind': 'file'
 924                 },
 925                 '/var/lib/cwl/workflow.json': {
 926                     'kind': 'json',
 927                     'content': {
 928                         'cwlVersion': 'v1.0',
 929                         '$graph': [
 930                             {
 931                                 'id': '#main',
 932                                 'inputs': [
 933                                     {'type': 'string', 'id': '#main/x'}
 934                                 ],
 935                                 'steps': [
 936                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 937                                      'run': '#submit_tool.cwl',
 938                                      'id': '#main/step1',
 939                                      'out': []}
 940                                 ],
 941                                 'class': 'Workflow',
 942                                 'outputs': []
 943                             },
 944                             {
 945                                 'inputs': [
 946                                     {
 947                                         'inputBinding': {'position': 1},
 948                                         'type': 'string',
 949                                         'id': '#submit_tool.cwl/x'}
 950                                 ],
 951                                 'requirements': [
 952                                     {
 953                                         'dockerPull': 'debian:8',
 954                                         'class': 'DockerRequirement',
 955                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 956                                     }
 957                                 ],
 958                                 'id': '#submit_tool.cwl',
 959                                 'outputs': [],
 960                                 'baseCommand': 'cat',
 961                                 'class': 'CommandLineTool'
 962                             }
 963                         ]
 964                     }
 965                 },
 966                 '/var/lib/cwl/cwl.input.json': {
 967                     'content': {
 968                         'x': 'XxX'
 969                     },
 970                     'kind': 'json'
 971                 }
 972             }, 'state': 'Committed',
 973             'output_path': '/var/spool/cwl',
 974             'name': 'a test workflow',
 975             'container_image': "999999999999999999999999999999d3+99",
 976             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 977                         '--no-log-timestamps', '--disable-validate',
 978                         '--eval-timeout=20', '--thread-count=1',
 979                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 980                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 981             'cwd': '/var/spool/cwl',
 982             'runtime_constraints': {
 983                 'API': True,
 984                 'vcpus': 1,
 985                 'ram': 1342177280
 986             },
 987             'use_existing': True,
 988             'properties': {
 989                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 990             },
 991             'secret_mounts': {}
 992         }
 993
 994         stubs.api.container_requests().create.assert_called_with(
 995             body=JsonDiffMatcher(expect_container))
 996         self.assertEqual(stubs.capture_stdout.getvalue(),
 997                          stubs.expect_container_request_uuid + '\n')
 998         self.assertEqual(exited, 0)
 999
1000     @stubs
1001     def test_submit_container_name(self, stubs):
1002         exited = arvados_cwl.main(
1003             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
1004                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1005             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1006
1007         expect_container = copy.deepcopy(stubs.expect_container_spec)
1008         expect_container["name"] = "hello container 123"
1009
1010         stubs.api.container_requests().create.assert_called_with(
1011             body=JsonDiffMatcher(expect_container))
1012         self.assertEqual(stubs.capture_stdout.getvalue(),
1013                          stubs.expect_container_request_uuid + '\n')
1014         self.assertEqual(exited, 0)
1015
1016     @stubs
1017     def test_submit_missing_input(self, stubs):
1018         exited = arvados_cwl.main(
1019             ["--submit", "--no-wait", "--api=containers", "--debug",
1020              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1021             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1022         self.assertEqual(exited, 0)
1023
1024         exited = arvados_cwl.main(
1025             ["--submit", "--no-wait", "--api=containers", "--debug",
1026              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
1027             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1028         self.assertEqual(exited, 1)
1029
1030     @stubs
1031     def test_submit_container_project(self, stubs):
1032         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1033         exited = arvados_cwl.main(
1034             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1035                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1036             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1037
1038         expect_container = copy.deepcopy(stubs.expect_container_spec)
1039         expect_container["owner_uuid"] = project_uuid
1040         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1041                                        '--no-log-timestamps', '--disable-validate',
1042                                        "--eval-timeout=20", "--thread-count=1",
1043                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
1044                                        '--on-error=continue',
1045                                        '--project-uuid='+project_uuid,
1046                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1047
1048         stubs.api.container_requests().create.assert_called_with(
1049             body=JsonDiffMatcher(expect_container))
1050         self.assertEqual(stubs.capture_stdout.getvalue(),
1051                          stubs.expect_container_request_uuid + '\n')
1052         self.assertEqual(exited, 0)
1053
1054     @stubs
1055     def test_submit_container_eval_timeout(self, stubs):
1056         exited = arvados_cwl.main(
1057             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1058                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1059             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1060
1061         expect_container = copy.deepcopy(stubs.expect_container_spec)
1062         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1063                                        '--no-log-timestamps', '--disable-validate',
1064                                        '--eval-timeout=60.0', '--thread-count=1',
1065                                        '--enable-reuse', "--collection-cache-size=256",
1066                                        '--debug', '--on-error=continue',
1067                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1068
1069         stubs.api.container_requests().create.assert_called_with(
1070             body=JsonDiffMatcher(expect_container))
1071         self.assertEqual(stubs.capture_stdout.getvalue(),
1072                          stubs.expect_container_request_uuid + '\n')
1073         self.assertEqual(exited, 0)
1074
1075     @stubs
1076     def test_submit_container_collection_cache(self, stubs):
1077         exited = arvados_cwl.main(
1078             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
1079                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1080             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1081
1082         expect_container = copy.deepcopy(stubs.expect_container_spec)
1083         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1084                                        '--no-log-timestamps', '--disable-validate',
1085                                        '--eval-timeout=20', '--thread-count=1',
1086                                        '--enable-reuse', "--collection-cache-size=500",
1087                                        '--debug', '--on-error=continue',
1088                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1089         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1090
1091         stubs.api.container_requests().create.assert_called_with(
1092             body=JsonDiffMatcher(expect_container))
1093         self.assertEqual(stubs.capture_stdout.getvalue(),
1094                          stubs.expect_container_request_uuid + '\n')
1095         self.assertEqual(exited, 0)
1096
1097     @stubs
1098     def test_submit_container_thread_count(self, stubs):
1099         exited = arvados_cwl.main(
1100             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1101                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1102             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1103
1104         expect_container = copy.deepcopy(stubs.expect_container_spec)
1105         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1106                                        '--no-log-timestamps', '--disable-validate',
1107                                        '--eval-timeout=20', '--thread-count=20',
1108                                        '--enable-reuse', "--collection-cache-size=256",
1109                                        '--debug', '--on-error=continue',
1110                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1111
1112         stubs.api.container_requests().create.assert_called_with(
1113             body=JsonDiffMatcher(expect_container))
1114         self.assertEqual(stubs.capture_stdout.getvalue(),
1115                          stubs.expect_container_request_uuid + '\n')
1116         self.assertEqual(exited, 0)
1117
1118     @stubs
1119     def test_submit_job_runner_image(self, stubs):
1120         exited = arvados_cwl.main(
1121             ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1122                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1123             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1124
1125         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
1126
1127         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1128         stubs.api.pipeline_instances().create.assert_called_with(
1129             body=JsonDiffMatcher(expect_pipeline))
1130         self.assertEqual(stubs.capture_stdout.getvalue(),
1131                          stubs.expect_pipeline_uuid + '\n')
1132         self.assertEqual(exited, 0)
1133
1134     @stubs
1135     def test_submit_container_runner_image(self, stubs):
1136         exited = arvados_cwl.main(
1137             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1138                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1139             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1140
1141         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1142
1143         expect_container = copy.deepcopy(stubs.expect_container_spec)
1144         stubs.api.container_requests().create.assert_called_with(
1145             body=JsonDiffMatcher(expect_container))
1146         self.assertEqual(stubs.capture_stdout.getvalue(),
1147                          stubs.expect_container_request_uuid + '\n')
1148         self.assertEqual(exited, 0)
1149
1150     @stubs
1151     def test_submit_priority(self, stubs):
1152         exited = arvados_cwl.main(
1153             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1154                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1155             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1156
1157         stubs.expect_container_spec["priority"] = 669
1158
1159         expect_container = copy.deepcopy(stubs.expect_container_spec)
1160         stubs.api.container_requests().create.assert_called_with(
1161             body=JsonDiffMatcher(expect_container))
1162         self.assertEqual(stubs.capture_stdout.getvalue(),
1163                          stubs.expect_container_request_uuid + '\n')
1164         self.assertEqual(exited, 0)
1165
1166     @stubs
1167     def test_submit_wf_runner_resources(self, stubs):
1168         exited = arvados_cwl.main(
1169             ["--submit", "--no-wait", "--api=containers", "--debug",
1170                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1171             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1172
1173         expect_container = copy.deepcopy(stubs.expect_container_spec)
1174         expect_container["runtime_constraints"] = {
1175             "API": True,
1176             "vcpus": 2,
1177             "ram": (2000+512) * 2**20
1178         }
1179         expect_container["name"] = "submit_wf_runner_resources.cwl"
1180         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1181             {
1182                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1183                 "coresMin": 2,
1184                 "ramMin": 2000,
1185                 "keep_cache": 512
1186             }
1187         ]
1188         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1189             "arv": "http://arvados.org/cwl#",
1190         }
1191         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1192                         '--no-log-timestamps', '--disable-validate',
1193                         '--eval-timeout=20', '--thread-count=1',
1194                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1195                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1196
1197         stubs.api.container_requests().create.assert_called_with(
1198             body=JsonDiffMatcher(expect_container))
1199         self.assertEqual(stubs.capture_stdout.getvalue(),
1200                          stubs.expect_container_request_uuid + '\n')
1201         self.assertEqual(exited, 0)
1202
1203     def tearDown(self):
1204         arvados_cwl.arvdocker.arv_docker_clear_cache()
1205
1206     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1207     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1208     @mock.patch("arvados.api")
1209     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1210         arvados_cwl.arvdocker.arv_docker_clear_cache()
1211
1212         arvrunner = mock.MagicMock()
1213         arvrunner.project_uuid = ""
1214         api.return_value = mock.MagicMock()
1215         arvrunner.api = api.return_value
1216         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1217                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1218                                                                         "link_class": "docker_image_repo+tag",
1219                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1220                                                                         "owner_uuid": "",
1221                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1222                                                             {"items": [{"created_at": "",
1223                                                                         "head_uuid": "",
1224                                                                         "link_class": "docker_image_hash",
1225                                                                         "name": "123456",
1226                                                                         "owner_uuid": "",
1227                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1228         )
1229         find_one_image_hash.return_value = "123456"
1230
1231         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1232                                                                               "owner_uuid": "",
1233                                                                               "manifest_text": "",
1234                                                                               "properties": ""
1235                                                                           }], "items_available": 1, "offset": 0},)
1236         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1237         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1238                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1239         self.assertEqual("9999999999999999999999999999999b+99",
1240                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1241
1242
1243     @stubs
1244     def test_submit_secrets(self, stubs):
1245         exited = arvados_cwl.main(
1246             ["--submit", "--no-wait", "--api=containers", "--debug",
1247                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1248             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1249
1250         expect_container = {
1251             "command": [
1252                 "arvados-cwl-runner",
1253                 "--local",
1254                 "--api=containers",
1255                 "--no-log-timestamps",
1256                 "--disable-validate",
1257                 "--eval-timeout=20",
1258                 '--thread-count=1',
1259                 "--enable-reuse",
1260                 "--collection-cache-size=256",
1261                 '--debug',
1262                 "--on-error=continue",
1263                 "/var/lib/cwl/workflow.json#main",
1264                 "/var/lib/cwl/cwl.input.json"
1265             ],
1266             "container_image": "999999999999999999999999999999d3+99",
1267             "cwd": "/var/spool/cwl",
1268             "mounts": {
1269                 "/var/lib/cwl/cwl.input.json": {
1270                     "content": {
1271                         "pw": {
1272                             "$include": "/secrets/s0"
1273                         }
1274                     },
1275                     "kind": "json"
1276                 },
1277                 "/var/lib/cwl/workflow.json": {
1278                     "content": {
1279                         "$graph": [
1280                             {
1281                                 "$namespaces": {
1282                                     "cwltool": "http://commonwl.org/cwltool#"
1283                                 },
1284                                 "arguments": [
1285                                     "md5sum",
1286                                     "example.conf"
1287                                 ],
1288                                 "class": "CommandLineTool",
1289                                 "hints": [
1290                                     {
1291                                         "class": "http://commonwl.org/cwltool#Secrets",
1292                                         "secrets": [
1293                                             "#secret_job.cwl/pw"
1294                                         ]
1295                                     }
1296                                 ],
1297                                 "id": "#secret_job.cwl",
1298                                 "inputs": [
1299                                     {
1300                                         "id": "#secret_job.cwl/pw",
1301                                         "type": "string"
1302                                     }
1303                                 ],
1304                                 "outputs": [
1305                                     {
1306                                         "id": "#secret_job.cwl/out",
1307                                         "type": "stdout"
1308                                     }
1309                                 ],
1310                                 "stdout": "hashed_example.txt",
1311                                 "requirements": [
1312                                     {
1313                                         "class": "InitialWorkDirRequirement",
1314                                         "listing": [
1315                                             {
1316                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1317                                                 "entryname": "example.conf"
1318                                             }
1319                                         ]
1320                                     }
1321                                 ]
1322                             },
1323                             {
1324                                 "class": "Workflow",
1325                                 "hints": [
1326                                     {
1327                                         "class": "DockerRequirement",
1328                                         "dockerPull": "debian:8",
1329                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1330                                     },
1331                                     {
1332                                         "class": "http://commonwl.org/cwltool#Secrets",
1333                                         "secrets": [
1334                                             "#main/pw"
1335                                         ]
1336                                     }
1337                                 ],
1338                                 "id": "#main",
1339                                 "inputs": [
1340                                     {
1341                                         "id": "#main/pw",
1342                                         "type": "string"
1343                                     }
1344                                 ],
1345                                 "outputs": [
1346                                     {
1347                                         "id": "#main/out",
1348                                         "outputSource": "#main/step1/out",
1349                                         "type": "File"
1350                                     }
1351                                 ],
1352                                 "steps": [
1353                                     {
1354                                         "id": "#main/step1",
1355                                         "in": [
1356                                             {
1357                                                 "id": "#main/step1/pw",
1358                                                 "source": "#main/pw"
1359                                             }
1360                                         ],
1361                                         "out": [
1362                                             "#main/step1/out"
1363                                         ],
1364                                         "run": "#secret_job.cwl"
1365                                     }
1366                                 ]
1367                             }
1368                         ],
1369                         "cwlVersion": "v1.0"
1370                     },
1371                     "kind": "json"
1372                 },
1373                 "/var/spool/cwl": {
1374                     "kind": "collection",
1375                     "writable": True
1376                 },
1377                 "stdout": {
1378                     "kind": "file",
1379                     "path": "/var/spool/cwl/cwl.output.json"
1380                 }
1381             },
1382             "name": "secret_wf.cwl",
1383             "output_path": "/var/spool/cwl",
1384             "priority": 500,
1385             "properties": {},
1386             "runtime_constraints": {
1387                 "API": True,
1388                 "ram": 1342177280,
1389                 "vcpus": 1
1390             },
1391             "secret_mounts": {
1392                 "/secrets/s0": {
1393                     "content": "blorp",
1394                     "kind": "text"
1395                 }
1396             },
1397             "state": "Committed",
1398             "use_existing": True
1399         }
1400
1401         stubs.api.container_requests().create.assert_called_with(
1402             body=JsonDiffMatcher(expect_container))
1403         self.assertEqual(stubs.capture_stdout.getvalue(),
1404                          stubs.expect_container_request_uuid + '\n')
1405         self.assertEqual(exited, 0)
1406
1407     @stubs
1408     def test_submit_request_uuid(self, stubs):
1409         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1410         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1411
1412         stubs.api.container_requests().update().execute.return_value = {
1413             "uuid": stubs.expect_container_request_uuid,
1414             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1415             "state": "Queued"
1416         }
1417
1418         exited = arvados_cwl.main(
1419             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1420                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1421             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1422
1423         stubs.api.container_requests().update.assert_called_with(
1424             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1425         self.assertEqual(stubs.capture_stdout.getvalue(),
1426                          stubs.expect_container_request_uuid + '\n')
1427         self.assertEqual(exited, 0)
1428
1429     @stubs
1430     def test_submit_container_cluster_id(self, stubs):
1431         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1432
1433         exited = arvados_cwl.main(
1434             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1435                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1436             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1437
1438         expect_container = copy.deepcopy(stubs.expect_container_spec)
1439
1440         stubs.api.container_requests().create.assert_called_with(
1441             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1442         self.assertEqual(stubs.capture_stdout.getvalue(),
1443                          stubs.expect_container_request_uuid + '\n')
1444         self.assertEqual(exited, 0)
1445
1446     @stubs
1447     def test_submit_validate_cluster_id(self, stubs):
1448         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1449         exited = arvados_cwl.main(
1450             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1451              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1452             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1453         self.assertEqual(exited, 1)
1454
1455     @mock.patch("arvados.collection.CollectionReader")
1456     @stubs
1457     def test_submit_uuid_inputs(self, stubs, collectionReader):
1458         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1459         def list_side_effect(**kwargs):
1460             m = mock.MagicMock()
1461             if "count" in kwargs:
1462                 m.execute.return_value = {"items": [
1463                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1464                 ]}
1465             else:
1466                 m.execute.return_value = {"items": []}
1467             return m
1468         stubs.api.collections().list.side_effect = list_side_effect
1469
1470         exited = arvados_cwl.main(
1471             ["--submit", "--no-wait", "--api=containers", "--debug",
1472                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1473             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1474
1475         expect_container = copy.deepcopy(stubs.expect_container_spec)
1476         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1477         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1478         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1479
1480         stubs.api.collections().list.assert_has_calls([
1481             mock.call(count='none',
1482                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1483                       select=['uuid', 'portable_data_hash'])])
1484         stubs.api.container_requests().create.assert_called_with(
1485             body=JsonDiffMatcher(expect_container))
1486         self.assertEqual(stubs.capture_stdout.getvalue(),
1487                          stubs.expect_container_request_uuid + '\n')
1488         self.assertEqual(exited, 0)
1489
1490     @stubs
1491     def test_submit_mismatched_uuid_inputs(self, stubs):
1492         def list_side_effect(**kwargs):
1493             m = mock.MagicMock()
1494             if "count" in kwargs:
1495                 m.execute.return_value = {"items": [
1496                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1497                 ]}
1498             else:
1499                 m.execute.return_value = {"items": []}
1500             return m
1501         stubs.api.collections().list.side_effect = list_side_effect
1502
1503         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1504             capture_stderr = StringIO()
1505             cwltool_logger = logging.getLogger('cwltool')
1506             stderr_logger = logging.StreamHandler(capture_stderr)
1507             cwltool_logger.addHandler(stderr_logger)
1508
1509             try:
1510                 exited = arvados_cwl.main(
1511                     ["--submit", "--no-wait", "--api=containers", "--debug",
1512                         "tests/wf/submit_wf.cwl", infile],
1513                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1514
1515                 self.assertEqual(exited, 1)
1516                 self.assertRegexpMatches(
1517                     capture_stderr.getvalue(),
1518                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1519             finally:
1520                 cwltool_logger.removeHandler(stderr_logger)
1521
1522     @mock.patch("arvados.collection.CollectionReader")
1523     @stubs
1524     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1525         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1526         capture_stderr = StringIO()
1527
1528         cwltool_logger = logging.getLogger('cwltool')
1529         stderr_logger = logging.StreamHandler(capture_stderr)
1530         cwltool_logger.addHandler(stderr_logger)
1531
1532         exited = arvados_cwl.main(
1533             ["--submit", "--no-wait", "--api=containers", "--debug",
1534                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1535             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1536
1537         try:
1538             self.assertEqual(exited, 1)
1539             self.assertRegexpMatches(
1540                 capture_stderr.getvalue(),
1541                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1542         finally:
1543             cwltool_logger.removeHandler(stderr_logger)
1544
1545
1546 class TestCreateTemplate(unittest.TestCase):
1547     existing_template_uuid = "zzzzz-p5p6p-validworkfloyml"
1548
1549     def _adjust_script_params(self, expect_component):
1550         expect_component['script_parameters']['x'] = {
1551             'dataclass': 'File',
1552             'required': True,
1553             'type': 'File',
1554             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1555         }
1556         expect_component['script_parameters']['y'] = {
1557             'dataclass': 'Collection',
1558             'required': True,
1559             'type': 'Directory',
1560             'value': '99999999999999999999999999999998+99',
1561         }
1562         expect_component['script_parameters']['z'] = {
1563             'dataclass': 'Collection',
1564             'required': True,
1565             'type': 'Directory',
1566         }
1567
1568     @stubs
1569     def test_create(self, stubs):
1570         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1571
1572         exited = arvados_cwl.main(
1573             ["--create-workflow", "--debug",
1574              "--api=jobs",
1575              "--project-uuid", project_uuid,
1576              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1577             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1578
1579         stubs.api.pipeline_instances().create.refute_called()
1580         stubs.api.jobs().create.refute_called()
1581
1582         expect_component = copy.deepcopy(stubs.expect_job_spec)
1583         self._adjust_script_params(expect_component)
1584         expect_template = {
1585             "components": {
1586                 "submit_wf.cwl": expect_component,
1587             },
1588             "name": "submit_wf.cwl",
1589             "owner_uuid": project_uuid,
1590         }
1591         stubs.api.pipeline_templates().create.assert_called_with(
1592             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1593
1594         self.assertEqual(stubs.capture_stdout.getvalue(),
1595                          stubs.expect_pipeline_template_uuid + '\n')
1596         self.assertEqual(exited, 0)
1597
1598     @stubs
1599     def test_create_name(self, stubs):
1600         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1601
1602         exited = arvados_cwl.main(
1603             ["--create-workflow", "--debug",
1604              "--project-uuid", project_uuid,
1605              "--api=jobs",
1606              "--name", "testing 123",
1607              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1608             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1609
1610         stubs.api.pipeline_instances().create.refute_called()
1611         stubs.api.jobs().create.refute_called()
1612
1613         expect_component = copy.deepcopy(stubs.expect_job_spec)
1614         self._adjust_script_params(expect_component)
1615         expect_template = {
1616             "components": {
1617                 "testing 123": expect_component,
1618             },
1619             "name": "testing 123",
1620             "owner_uuid": project_uuid,
1621         }
1622         stubs.api.pipeline_templates().create.assert_called_with(
1623             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1624
1625         self.assertEqual(stubs.capture_stdout.getvalue(),
1626                          stubs.expect_pipeline_template_uuid + '\n')
1627         self.assertEqual(exited, 0)
1628
1629     @stubs
1630     def test_update_name(self, stubs):
1631         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1632
1633         exited = arvados_cwl.main(
1634             ["--update-workflow", self.existing_template_uuid,
1635              "--debug",
1636              "--project-uuid", project_uuid,
1637              "--api=jobs",
1638              "--name", "testing 123",
1639              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1640             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1641
1642         stubs.api.pipeline_instances().create.refute_called()
1643         stubs.api.jobs().create.refute_called()
1644
1645         expect_component = copy.deepcopy(stubs.expect_job_spec)
1646         self._adjust_script_params(expect_component)
1647         expect_template = {
1648             "components": {
1649                 "testing 123": expect_component,
1650             },
1651             "name": "testing 123",
1652             "owner_uuid": project_uuid,
1653         }
1654         stubs.api.pipeline_templates().create.refute_called()
1655         stubs.api.pipeline_templates().update.assert_called_with(
1656             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1657
1658         self.assertEqual(stubs.capture_stdout.getvalue(),
1659                          self.existing_template_uuid + '\n')
1660         self.assertEqual(exited, 0)
1661
1662
1663 class TestCreateWorkflow(unittest.TestCase):
1664     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1665     expect_workflow = StripYAMLComments(
1666         open("tests/wf/expect_packed.cwl").read())
1667
1668     @stubs
1669     def test_create(self, stubs):
1670         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1671
1672         exited = arvados_cwl.main(
1673             ["--create-workflow", "--debug",
1674              "--api=containers",
1675              "--project-uuid", project_uuid,
1676              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1677             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1678
1679         stubs.api.pipeline_templates().create.refute_called()
1680         stubs.api.container_requests().create.refute_called()
1681
1682         body = {
1683             "workflow": {
1684                 "owner_uuid": project_uuid,
1685                 "name": "submit_wf.cwl",
1686                 "description": "",
1687                 "definition": self.expect_workflow,
1688             }
1689         }
1690         stubs.api.workflows().create.assert_called_with(
1691             body=JsonDiffMatcher(body))
1692
1693         self.assertEqual(stubs.capture_stdout.getvalue(),
1694                          stubs.expect_workflow_uuid + '\n')
1695         self.assertEqual(exited, 0)
1696
1697     @stubs
1698     def test_create_name(self, stubs):
1699         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1700
1701         exited = arvados_cwl.main(
1702             ["--create-workflow", "--debug",
1703              "--api=containers",
1704              "--project-uuid", project_uuid,
1705              "--name", "testing 123",
1706              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1707             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1708
1709         stubs.api.pipeline_templates().create.refute_called()
1710         stubs.api.container_requests().create.refute_called()
1711
1712         body = {
1713             "workflow": {
1714                 "owner_uuid": project_uuid,
1715                 "name": "testing 123",
1716                 "description": "",
1717                 "definition": self.expect_workflow,
1718             }
1719         }
1720         stubs.api.workflows().create.assert_called_with(
1721             body=JsonDiffMatcher(body))
1722
1723         self.assertEqual(stubs.capture_stdout.getvalue(),
1724                          stubs.expect_workflow_uuid + '\n')
1725         self.assertEqual(exited, 0)
1726
1727     @stubs
1728     def test_incompatible_api(self, stubs):
1729         capture_stderr = StringIO()
1730         acr_logger = logging.getLogger('arvados.cwl-runner')
1731         stderr_logger = logging.StreamHandler(capture_stderr)
1732         acr_logger.addHandler(stderr_logger)
1733
1734         try:
1735             exited = arvados_cwl.main(
1736                 ["--update-workflow", self.existing_workflow_uuid,
1737                  "--api=jobs",
1738                  "--debug",
1739                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1740                 sys.stderr, sys.stderr, api_client=stubs.api)
1741             self.assertEqual(exited, 1)
1742             self.assertRegexpMatches(
1743                 capture_stderr.getvalue(),
1744                 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1745         finally:
1746             acr_logger.removeHandler(stderr_logger)
1747
1748     @stubs
1749     def test_update(self, stubs):
1750         exited = arvados_cwl.main(
1751             ["--update-workflow", self.existing_workflow_uuid,
1752              "--debug",
1753              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1754             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1755
1756         body = {
1757             "workflow": {
1758                 "name": "submit_wf.cwl",
1759                 "description": "",
1760                 "definition": self.expect_workflow,
1761             }
1762         }
1763         stubs.api.workflows().update.assert_called_with(
1764             uuid=self.existing_workflow_uuid,
1765             body=JsonDiffMatcher(body))
1766         self.assertEqual(stubs.capture_stdout.getvalue(),
1767                          self.existing_workflow_uuid + '\n')
1768         self.assertEqual(exited, 0)
1769
1770     @stubs
1771     def test_update_name(self, stubs):
1772         exited = arvados_cwl.main(
1773             ["--update-workflow", self.existing_workflow_uuid,
1774              "--debug", "--name", "testing 123",
1775              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1776             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1777
1778         body = {
1779             "workflow": {
1780                 "name": "testing 123",
1781                 "description": "",
1782                 "definition": self.expect_workflow,
1783             }
1784         }
1785         stubs.api.workflows().update.assert_called_with(
1786             uuid=self.existing_workflow_uuid,
1787             body=JsonDiffMatcher(body))
1788         self.assertEqual(stubs.capture_stdout.getvalue(),
1789                          self.existing_workflow_uuid + '\n')
1790         self.assertEqual(exited, 0)
1791
1792     @stubs
1793     def test_create_collection_per_tool(self, stubs):
1794         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1795
1796         exited = arvados_cwl.main(
1797             ["--create-workflow", "--debug",
1798              "--api=containers",
1799              "--project-uuid", project_uuid,
1800              "tests/collection_per_tool/collection_per_tool.cwl"],
1801             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1802
1803         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1804         expect_workflow = StripYAMLComments(open(toolfile).read())
1805
1806         body = {
1807             "workflow": {
1808                 "owner_uuid": project_uuid,
1809                 "name": "collection_per_tool.cwl",
1810                 "description": "",
1811                 "definition": expect_workflow,
1812             }
1813         }
1814         stubs.api.workflows().create.assert_called_with(
1815             body=JsonDiffMatcher(body))
1816
1817         self.assertEqual(stubs.capture_stdout.getvalue(),
1818                          stubs.expect_workflow_uuid + '\n')
1819         self.assertEqual(exited, 0)
1820
1821 class TestTemplateInputs(unittest.TestCase):
1822     expect_template = {
1823         "components": {
1824             "inputs_test.cwl": {
1825                 'runtime_constraints': {
1826                     'docker_image': '999999999999999999999999999999d3+99',
1827                     'min_ram_mb_per_node': 1024
1828                 },
1829                 'script_parameters': {
1830                     'cwl:tool':
1831                     'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
1832                     'optionalFloatInput': None,
1833                     'fileInput': {
1834                         'type': 'File',
1835                         'dataclass': 'File',
1836                         'required': True,
1837                         'title': "It's a file; we expect to find some characters in it.",
1838                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1839                     },
1840                     'floatInput': {
1841                         'type': 'float',
1842                         'dataclass': 'number',
1843                         'required': True,
1844                         'title': 'Floats like a duck',
1845                         'default': 0.1,
1846                         'value': 0.1,
1847                     },
1848                     'optionalFloatInput': {
1849                         'type': ['null', 'float'],
1850                         'dataclass': 'number',
1851                         'required': False,
1852                     },
1853                     'boolInput': {
1854                         'type': 'boolean',
1855                         'dataclass': 'boolean',
1856                         'required': True,
1857                         'title': 'True or false?',
1858                     },
1859                 },
1860                 'repository': 'arvados',
1861                 'script_version': 'master',
1862                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1863                 'script': 'cwl-runner',
1864             },
1865         },
1866         "name": "inputs_test.cwl",
1867     }
1868
1869     @stubs
1870     def test_inputs_empty(self, stubs):
1871         exited = arvados_cwl.main(
1872             ["--debug", "--api=jobs", "--create-template",
1873              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1874             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1875
1876         stubs.api.pipeline_templates().create.assert_called_with(
1877             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1878
1879         self.assertEqual(exited, 0)
1880
1881     @stubs
1882     def test_inputs(self, stubs):
1883         exited = arvados_cwl.main(
1884             ["--api=jobs", "--create-template",
1885              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1886             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1887
1888         expect_template = copy.deepcopy(self.expect_template)
1889         params = expect_template[
1890             "components"]["inputs_test.cwl"]["script_parameters"]
1891         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1892         params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
1893         params["floatInput"]["value"] = 1.234
1894         params["boolInput"]["value"] = True
1895
1896         stubs.api.pipeline_templates().create.assert_called_with(
1897             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1898         self.assertEqual(exited, 0)