sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20
  21 from io import BytesIO
  22
  23 # StringIO.StringIO and io.StringIO have different behavior write() is
  24 # called with both python2 (byte) strings and unicode strings
  25 # (specifically there's some logging in cwltool that causes trouble).
  26 # This isn't a problem on python3 because all string are unicode.
  27 if sys.version_info[0] < 3:
  28     from StringIO import StringIO
  29 else:
  30     from io import StringIO
  31
  32 import arvados
  33 import arvados.collection
  34 import arvados_cwl
  35 import arvados_cwl.executor
  36 import arvados_cwl.runner
  37 import arvados.keep
  38
  39 from .matcher import JsonDiffMatcher, StripYAMLComments
  40 from .mock_discovery import get_rootDesc
  41
  42 import ruamel.yaml as yaml
  43
  44 _rootDesc = None
  45
  46 def stubs(func):
  47     @functools.wraps(func)
  48     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  49     @mock.patch("arvados.collection.KeepClient")
  50     @mock.patch("arvados.keep.KeepClient")
  51     @mock.patch("arvados.events.subscribe")
  52     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  53         class Stubs(object):
  54             pass
  55         stubs = Stubs()
  56         stubs.events = events
  57         stubs.keepdocker = keepdocker
  58
  59         def putstub(p, **kwargs):
  60             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  61         keep_client1().put.side_effect = putstub
  62         keep_client1.put.side_effect = putstub
  63         keep_client2().put.side_effect = putstub
  64         keep_client2.put.side_effect = putstub
  65
  66         stubs.keep_client = keep_client2
  67         stubs.docker_images = {
  68             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  69             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  70             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  71             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  72         }
  73         def kd(a, b, image_name=None, image_tag=None):
  74             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  75         stubs.keepdocker.side_effect = kd
  76
  77         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  78         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  79
  80         if sys.version_info[0] < 3:
  81             stubs.capture_stdout = BytesIO()
  82         else:
  83             stubs.capture_stdout = StringIO()
  84
  85         stubs.api = mock.MagicMock()
  86         stubs.api._rootDesc = get_rootDesc()
  87
  88         stubs.api.users().current().execute.return_value = {
  89             "uuid": stubs.fake_user_uuid,
  90         }
  91         stubs.api.collections().list().execute.return_value = {"items": []}
  92         stubs.api.containers().current().execute.return_value = {
  93             "uuid": stubs.fake_container_uuid,
  94         }
  95
  96         class CollectionExecute(object):
  97             def __init__(self, exe):
  98                 self.exe = exe
  99             def execute(self, num_retries=None):
 100                 return self.exe
 101
 102         def collection_createstub(created_collections, body, ensure_unique_name=None):
 103             mt = body["manifest_text"].encode('utf-8')
 104             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 105             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 106             created_collections[uuid] = {
 107                 "uuid": uuid,
 108                 "portable_data_hash": pdh,
 109                 "manifest_text": mt.decode('utf-8')
 110             }
 111             return CollectionExecute(created_collections[uuid])
 112
 113         def collection_getstub(created_collections, uuid):
 114             for v in viewvalues(created_collections):
 115                 if uuid in (v["uuid"], v["portable_data_hash"]):
 116                     return CollectionExecute(v)
 117
 118         created_collections = {
 119             "99999999999999999999999999999998+99": {
 120                 "uuid": "",
 121                 "portable_data_hash": "99999999999999999999999999999998+99",
 122                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 123             },
 124             "99999999999999999999999999999997+99": {
 125                 "uuid": "",
 126                 "portable_data_hash": "99999999999999999999999999999997+99",
 127                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 128             },
 129             "99999999999999999999999999999994+99": {
 130                 "uuid": "",
 131                 "portable_data_hash": "99999999999999999999999999999994+99",
 132                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 133             },
 134             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 135                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 136                 "portable_data_hash": "999999999999999999999999999999d3+99",
 137                 "manifest_text": ""
 138             },
 139             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 140                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 141                 "portable_data_hash": "999999999999999999999999999999d4+99",
 142                 "manifest_text": ""
 143             },
 144             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 145                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 146                 "portable_data_hash": "999999999999999999999999999999d5+99",
 147                 "manifest_text": ""
 148             },
 149             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 150                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 151                 "portable_data_hash": "999999999999999999999999999999d6+99",
 152                 "manifest_text": ""
 153             }
 154         }
 155         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 156         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 157
 158         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 159         stubs.api.jobs().create().execute.return_value = {
 160             "uuid": stubs.expect_job_uuid,
 161             "state": "Queued",
 162         }
 163
 164         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 165         stubs.api.container_requests().create().execute.return_value = {
 166             "uuid": stubs.expect_container_request_uuid,
 167             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 168             "state": "Queued"
 169         }
 170
 171         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 172         stubs.api.pipeline_templates().create().execute.return_value = {
 173             "uuid": stubs.expect_pipeline_template_uuid,
 174         }
 175         stubs.expect_job_spec = {
 176             'runtime_constraints': {
 177                 'docker_image': '999999999999999999999999999999d3+99',
 178                 'min_ram_mb_per_node': 1024
 179             },
 180             'script_parameters': {
 181                 'x': {
 182                     'basename': 'blorp.txt',
 183                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 184                     'class': 'File'
 185                 },
 186                 'y': {
 187                     'basename': '99999999999999999999999999999998+99',
 188                     'location': 'keep:99999999999999999999999999999998+99',
 189                     'class': 'Directory'
 190                 },
 191                 'z': {
 192                     'basename': 'anonymous',
 193                     "listing": [{
 194                         "basename": "renamed.txt",
 195                         "class": "File",
 196                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 197                         "size": 0
 198                     }],
 199                     'class': 'Directory'
 200                 },
 201                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 202             },
 203             'repository': 'arvados',
 204             'script_version': 'master',
 205             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 206             'script': 'cwl-runner'
 207         }
 208         stubs.pipeline_component = stubs.expect_job_spec.copy()
 209         stubs.expect_pipeline_instance = {
 210             'name': 'submit_wf.cwl',
 211             'state': 'RunningOnServer',
 212             'owner_uuid': None,
 213             "components": {
 214                 "cwl-runner": {
 215                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 216                     'script_parameters': {
 217                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 218                         'x': {"value": {
 219                             'basename': 'blorp.txt',
 220                             'class': 'File',
 221                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         }},
 224                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 225                               'listing': [
 226                                   {
 227                                       'basename': 'renamed.txt',
 228                                       'class': 'File', 'location':
 229                                       'keep:99999999999999999999999999999998+99/file1.txt',
 230                                       'size': 0
 231                                   }
 232                               ]}},
 233                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 234                         'arv:debug': True,
 235                         'arv:enable_reuse': True,
 236                         'arv:on_error': 'continue'
 237                     },
 238                     'repository': 'arvados',
 239                     'script_version': 'master',
 240                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 241                     'script': 'cwl-runner',
 242                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 243                 }
 244             }
 245         }
 246         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 247         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 248         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 249         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 250         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 251             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 252             "state": "Queued"
 253         }
 254         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 255         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 256
 257         with open("tests/wf/submit_wf_packed.cwl") as f:
 258             expect_packed_workflow = yaml.round_trip_load(f)
 259
 260         stubs.expect_container_spec = {
 261             'priority': 500,
 262             'mounts': {
 263                 '/var/spool/cwl': {
 264                     'writable': True,
 265                     'kind': 'collection'
 266                 },
 267                 '/var/lib/cwl/workflow.json': {
 268                     'content': expect_packed_workflow,
 269                     'kind': 'json'
 270                 },
 271                 'stdout': {
 272                     'path': '/var/spool/cwl/cwl.output.json',
 273                     'kind': 'file'
 274                 },
 275                 '/var/lib/cwl/cwl.input.json': {
 276                     'kind': 'json',
 277                     'content': {
 278                         'y': {
 279                             'basename': '99999999999999999999999999999998+99',
 280                             'location': 'keep:99999999999999999999999999999998+99',
 281                             'class': 'Directory'},
 282                         'x': {
 283                             'basename': u'blorp.txt',
 284                             'class': 'File',
 285                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 286                             "size": 16
 287                         },
 288                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 289                             {'basename': 'renamed.txt',
 290                              'class': 'File',
 291                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 292                              'size': 0
 293                             }
 294                         ]}
 295                     },
 296                     'kind': 'json'
 297                 }
 298             },
 299             'secret_mounts': {},
 300             'state': 'Committed',
 301             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 302                         '--no-log-timestamps', '--disable-validate',
 303                         '--eval-timeout=20', '--thread-count=1',
 304                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 305                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 306             'name': 'submit_wf.cwl',
 307             'container_image': '999999999999999999999999999999d3+99',
 308             'output_path': '/var/spool/cwl',
 309             'cwd': '/var/spool/cwl',
 310             'runtime_constraints': {
 311                 'API': True,
 312                 'vcpus': 1,
 313                 'ram': (1024+256)*1024*1024
 314             },
 315             'use_existing': True,
 316             'properties': {},
 317             'secret_mounts': {}
 318         }
 319
 320         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 321         stubs.api.workflows().create().execute.return_value = {
 322             "uuid": stubs.expect_workflow_uuid,
 323         }
 324         def update_mock(**kwargs):
 325             stubs.updated_uuid = kwargs.get('uuid')
 326             return mock.DEFAULT
 327         stubs.api.workflows().update.side_effect = update_mock
 328         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 329             "uuid": stubs.updated_uuid,
 330         }
 331
 332         return func(self, stubs, *args, **kwargs)
 333     return wrapped
 334
 335
 336 class TestSubmit(unittest.TestCase):
 337     @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
 338     @mock.patch("time.sleep")
 339     @stubs
 340     def test_submit(self, stubs, tm, arvdock):
 341         def get_image(api_client, dockerRequirement, pull_image, project_uuid):
 342             if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
 343                 return '999999999999999999999999999999d3+99'
 344             elif dockerRequirement["dockerPull"] == "debian:8":
 345                 return '999999999999999999999999999999d4+99'
 346         arvdock.side_effect = get_image
 347
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--api=jobs", "--debug",
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 352
 353         stubs.api.collections().create.assert_has_calls([
 354             mock.call(body=JsonDiffMatcher({
 355                 'manifest_text':
 356                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 357                 'replication_desired': None,
 358                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 359             }), ensure_unique_name=False),
 360             mock.call(body=JsonDiffMatcher({
 361                 'manifest_text':
 362                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 363                 'replication_desired': None,
 364                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 365             }), ensure_unique_name=False),
 366             mock.call(body=JsonDiffMatcher({
 367                 'manifest_text':
 368                 ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
 369                 'replication_desired': None,
 370                 'name': 'submit_wf.cwl',
 371             }), ensure_unique_name=True)        ])
 372
 373         arvdock.assert_has_calls([
 374             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 375             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
 376             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 377         ])
 378
 379         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 380         stubs.api.pipeline_instances().create.assert_called_with(
 381             body=JsonDiffMatcher(expect_pipeline))
 382         self.assertEqual(stubs.capture_stdout.getvalue(),
 383                          stubs.expect_pipeline_uuid + '\n')
 384         self.assertEqual(exited, 0)
 385
 386     @mock.patch("time.sleep")
 387     @stubs
 388     def test_submit_no_reuse(self, stubs, tm):
 389         exited = arvados_cwl.main(
 390             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 391              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 392             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 393
 394         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 395         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 396         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 397
 398         stubs.api.pipeline_instances().create.assert_called_with(
 399             body=JsonDiffMatcher(expect_pipeline))
 400         self.assertEqual(stubs.capture_stdout.getvalue(),
 401                          stubs.expect_pipeline_uuid + '\n')
 402         self.assertEqual(exited, 0)
 403
 404     @stubs
 405     def test_error_when_multiple_storage_classes_specified(self, stubs):
 406         storage_classes = "foo,bar"
 407         exited = arvados_cwl.main(
 408                 ["--debug", "--storage-classes", storage_classes,
 409                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410                 sys.stdin, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 1)
 412
 413     @mock.patch("time.sleep")
 414     @stubs
 415     def test_submit_on_error(self, stubs, tm):
 416         exited = arvados_cwl.main(
 417             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 418              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 419             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 420
 421         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 422         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 423
 424         stubs.api.pipeline_instances().create.assert_called_with(
 425             body=JsonDiffMatcher(expect_pipeline))
 426         self.assertEqual(stubs.capture_stdout.getvalue(),
 427                          stubs.expect_pipeline_uuid + '\n')
 428         self.assertEqual(exited, 0)
 429
 430     @mock.patch("time.sleep")
 431     @stubs
 432     def test_submit_runner_ram(self, stubs, tm):
 433         exited = arvados_cwl.main(
 434             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 435              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 436             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 437
 438         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 439         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 440
 441         stubs.api.pipeline_instances().create.assert_called_with(
 442             body=JsonDiffMatcher(expect_pipeline))
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_pipeline_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @mock.patch("time.sleep")
 448     @stubs
 449     def test_submit_invalid_runner_ram(self, stubs, tm):
 450         exited = arvados_cwl.main(
 451             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 452              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 453             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 454         self.assertEqual(exited, 1)
 455
 456     @mock.patch("time.sleep")
 457     @stubs
 458     def test_submit_output_name(self, stubs, tm):
 459         output_name = "test_output_name"
 460
 461         exited = arvados_cwl.main(
 462             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 463              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 464             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 465
 466         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 467         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 468
 469         stubs.api.pipeline_instances().create.assert_called_with(
 470             body=JsonDiffMatcher(expect_pipeline))
 471         self.assertEqual(stubs.capture_stdout.getvalue(),
 472                          stubs.expect_pipeline_uuid + '\n')
 473         self.assertEqual(exited, 0)
 474
 475     @mock.patch("time.sleep")
 476     @stubs
 477     def test_submit_pipeline_name(self, stubs, tm):
 478         exited = arvados_cwl.main(
 479             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 480              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 481             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 482         self.assertEqual(exited, 0)
 483
 484         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 485         expect_pipeline["name"] = "hello job 123"
 486
 487         stubs.api.pipeline_instances().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_pipeline))
 489         self.assertEqual(stubs.capture_stdout.getvalue(),
 490                          stubs.expect_pipeline_uuid + '\n')
 491
 492     @mock.patch("time.sleep")
 493     @stubs
 494     def test_submit_output_tags(self, stubs, tm):
 495         output_tags = "tag0,tag1,tag2"
 496
 497         exited = arvados_cwl.main(
 498             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 499              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 500             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 501         self.assertEqual(exited, 0)
 502
 503         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 504         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 505
 506         stubs.api.pipeline_instances().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_pipeline))
 508         self.assertEqual(stubs.capture_stdout.getvalue(),
 509                          stubs.expect_pipeline_uuid + '\n')
 510
 511     @mock.patch("time.sleep")
 512     @stubs
 513     def test_submit_with_project_uuid(self, stubs, tm):
 514         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--debug",
 518              "--project-uuid", project_uuid,
 519              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 520             sys.stdout, sys.stderr, api_client=stubs.api)
 521         self.assertEqual(exited, 0)
 522
 523         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 524         expect_pipeline["owner_uuid"] = project_uuid
 525         stubs.api.pipeline_instances().create.assert_called_with(
 526             body=JsonDiffMatcher(expect_pipeline))
 527
 528     @stubs
 529     def test_submit_container(self, stubs):
 530         exited = arvados_cwl.main(
 531             ["--submit", "--no-wait", "--api=containers", "--debug",
 532                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 533             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 534
 535         stubs.api.collections().create.assert_has_calls([
 536             mock.call(body=JsonDiffMatcher({
 537                 'manifest_text':
 538                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 539                 'replication_desired': None,
 540                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 541             }), ensure_unique_name=False),
 542             mock.call(body=JsonDiffMatcher({
 543                 'manifest_text':
 544                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 545                 'replication_desired': None,
 546                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 547             }), ensure_unique_name=False)])
 548
 549         expect_container = copy.deepcopy(stubs.expect_container_spec)
 550         stubs.api.container_requests().create.assert_called_with(
 551             body=JsonDiffMatcher(expect_container))
 552         self.assertEqual(stubs.capture_stdout.getvalue(),
 553                          stubs.expect_container_request_uuid + '\n')
 554         self.assertEqual(exited, 0)
 555
 556     @stubs
 557     def test_submit_container_no_reuse(self, stubs):
 558         exited = arvados_cwl.main(
 559             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 560                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 561             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 562
 563         expect_container = copy.deepcopy(stubs.expect_container_spec)
 564         expect_container["command"] = [
 565             'arvados-cwl-runner', '--local', '--api=containers',
 566             '--no-log-timestamps', '--disable-validate',
 567             '--eval-timeout=20', '--thread-count=1',
 568             '--disable-reuse', "--collection-cache-size=256",
 569             '--debug', '--on-error=continue',
 570             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 571         expect_container["use_existing"] = False
 572
 573         stubs.api.container_requests().create.assert_called_with(
 574             body=JsonDiffMatcher(expect_container))
 575         self.assertEqual(stubs.capture_stdout.getvalue(),
 576                          stubs.expect_container_request_uuid + '\n')
 577         self.assertEqual(exited, 0)
 578
 579     @stubs
 580     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 581         exited = arvados_cwl.main(
 582             ["--submit", "--no-wait", "--api=containers", "--debug",
 583              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 584             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 585         self.assertEqual(exited, 0)
 586
 587         expect_container = copy.deepcopy(stubs.expect_container_spec)
 588         expect_container["command"] = [
 589             'arvados-cwl-runner', '--local', '--api=containers',
 590             '--no-log-timestamps', '--disable-validate',
 591             '--eval-timeout=20', '--thread-count=1',
 592             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 593             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 594         expect_container["use_existing"] = False
 595         expect_container["name"] = "submit_wf_no_reuse.cwl"
 596         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 597             {
 598                 "class": "http://arvados.org/cwl#ReuseRequirement",
 599                 "enableReuse": False,
 600             },
 601         ]
 602         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 603             "arv": "http://arvados.org/cwl#",
 604             "cwltool": "http://commonwl.org/cwltool#"
 605         }
 606
 607         stubs.api.container_requests().create.assert_called_with(
 608             body=JsonDiffMatcher(expect_container))
 609         self.assertEqual(stubs.capture_stdout.getvalue(),
 610                          stubs.expect_container_request_uuid + '\n')
 611
 612
 613     @stubs
 614     def test_submit_container_on_error(self, stubs):
 615         exited = arvados_cwl.main(
 616             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 617                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 618             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 619
 620         expect_container = copy.deepcopy(stubs.expect_container_spec)
 621         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 622                                        '--no-log-timestamps', '--disable-validate',
 623                                        '--eval-timeout=20', '--thread-count=1',
 624                                        '--enable-reuse', "--collection-cache-size=256",
 625                                        '--debug', '--on-error=stop',
 626                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(stubs.capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632         self.assertEqual(exited, 0)
 633
 634     @stubs
 635     def test_submit_container_output_name(self, stubs):
 636         output_name = "test_output_name"
 637
 638         exited = arvados_cwl.main(
 639             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 640                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 641             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 642
 643         expect_container = copy.deepcopy(stubs.expect_container_spec)
 644         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 645                                        '--no-log-timestamps', '--disable-validate',
 646                                        '--eval-timeout=20', '--thread-count=1',
 647                                        '--enable-reuse', "--collection-cache-size=256",
 648                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 649                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 650         expect_container["output_name"] = output_name
 651
 652         stubs.api.container_requests().create.assert_called_with(
 653             body=JsonDiffMatcher(expect_container))
 654         self.assertEqual(stubs.capture_stdout.getvalue(),
 655                          stubs.expect_container_request_uuid + '\n')
 656         self.assertEqual(exited, 0)
 657
 658     @stubs
 659     def test_submit_storage_classes(self, stubs):
 660         exited = arvados_cwl.main(
 661             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 662                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 663             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 664
 665         expect_container = copy.deepcopy(stubs.expect_container_spec)
 666         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 667                                        '--no-log-timestamps', '--disable-validate',
 668                                        '--eval-timeout=20', '--thread-count=1',
 669                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 670                                        "--storage-classes=foo", '--on-error=continue',
 671                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 672
 673         stubs.api.container_requests().create.assert_called_with(
 674             body=JsonDiffMatcher(expect_container))
 675         self.assertEqual(stubs.capture_stdout.getvalue(),
 676                          stubs.expect_container_request_uuid + '\n')
 677         self.assertEqual(exited, 0)
 678
 679     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 680     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 681     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 682     @stubs
 683     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 684         def set_final_output(job_order, output_callback, runtimeContext):
 685             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 686             return []
 687         job.side_effect = set_final_output
 688
 689         exited = arvados_cwl.main(
 690             ["--debug", "--local", "--storage-classes=foo",
 691                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 692             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 693
 694         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 695         self.assertEqual(exited, 0)
 696
 697     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 698     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 699     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 700     @stubs
 701     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 702         def set_final_output(job_order, output_callback, runtimeContext):
 703             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 704             return []
 705         job.side_effect = set_final_output
 706
 707         exited = arvados_cwl.main(
 708             ["--debug", "--local",
 709                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 710             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 711
 712         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 713         self.assertEqual(exited, 0)
 714
 715     @stubs
 716     def test_submit_container_output_ttl(self, stubs):
 717         exited = arvados_cwl.main(
 718             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 719                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 720             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 721
 722         expect_container = copy.deepcopy(stubs.expect_container_spec)
 723         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 724                                        '--no-log-timestamps', '--disable-validate',
 725                                        '--eval-timeout=20', '--thread-count=1',
 726                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 727                                        '--on-error=continue',
 728                                        "--intermediate-output-ttl=3600",
 729                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 730
 731         stubs.api.container_requests().create.assert_called_with(
 732             body=JsonDiffMatcher(expect_container))
 733         self.assertEqual(stubs.capture_stdout.getvalue(),
 734                          stubs.expect_container_request_uuid + '\n')
 735         self.assertEqual(exited, 0)
 736
 737     @stubs
 738     def test_submit_container_trash_intermediate(self, stubs):
 739         exited = arvados_cwl.main(
 740             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 741                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 742             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 743
 744
 745         expect_container = copy.deepcopy(stubs.expect_container_spec)
 746         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 747                                        '--no-log-timestamps', '--disable-validate',
 748                                        '--eval-timeout=20', '--thread-count=1',
 749                                        '--enable-reuse', "--collection-cache-size=256",
 750                                        '--debug', '--on-error=continue',
 751                                        "--trash-intermediate",
 752                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 753
 754         stubs.api.container_requests().create.assert_called_with(
 755             body=JsonDiffMatcher(expect_container))
 756         self.assertEqual(stubs.capture_stdout.getvalue(),
 757                          stubs.expect_container_request_uuid + '\n')
 758         self.assertEqual(exited, 0)
 759
 760     @stubs
 761     def test_submit_container_output_tags(self, stubs):
 762         output_tags = "tag0,tag1,tag2"
 763
 764         exited = arvados_cwl.main(
 765             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 766                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 767             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 768
 769         expect_container = copy.deepcopy(stubs.expect_container_spec)
 770         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 771                                        '--no-log-timestamps', '--disable-validate',
 772                                        '--eval-timeout=20', '--thread-count=1',
 773                                        '--enable-reuse', "--collection-cache-size=256",
 774                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 775                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 776
 777         stubs.api.container_requests().create.assert_called_with(
 778             body=JsonDiffMatcher(expect_container))
 779         self.assertEqual(stubs.capture_stdout.getvalue(),
 780                          stubs.expect_container_request_uuid + '\n')
 781         self.assertEqual(exited, 0)
 782
 783     @stubs
 784     def test_submit_container_runner_ram(self, stubs):
 785         exited = arvados_cwl.main(
 786             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 787                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 788             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 789
 790         expect_container = copy.deepcopy(stubs.expect_container_spec)
 791         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 792
 793         stubs.api.container_requests().create.assert_called_with(
 794             body=JsonDiffMatcher(expect_container))
 795         self.assertEqual(stubs.capture_stdout.getvalue(),
 796                          stubs.expect_container_request_uuid + '\n')
 797         self.assertEqual(exited, 0)
 798
 799     @mock.patch("arvados.collection.CollectionReader")
 800     @mock.patch("time.sleep")
 801     @stubs
 802     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 803         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 804         exited = arvados_cwl.main(
 805             ["--submit", "--no-wait", "--api=containers", "--debug",
 806              "tests/wf/submit_keepref_wf.cwl"],
 807             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 808         self.assertEqual(exited, 0)
 809
 810     @mock.patch("arvados.collection.CollectionReader")
 811     @mock.patch("time.sleep")
 812     @stubs
 813     def test_submit_keepref(self, stubs, tm, reader):
 814         with open("tests/wf/expect_arvworkflow.cwl") as f:
 815             reader().open().__enter__().read.return_value = f.read()
 816
 817         exited = arvados_cwl.main(
 818             ["--submit", "--no-wait", "--api=containers", "--debug",
 819              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 820             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 821
 822         expect_container = {
 823             'priority': 500,
 824             'mounts': {
 825                 '/var/spool/cwl': {
 826                     'writable': True,
 827                     'kind': 'collection'
 828                 },
 829                 'stdout': {
 830                     'path': '/var/spool/cwl/cwl.output.json',
 831                     'kind': 'file'
 832                 },
 833                 '/var/lib/cwl/workflow': {
 834                     'portable_data_hash': '99999999999999999999999999999994+99',
 835                     'kind': 'collection'
 836                 },
 837                 '/var/lib/cwl/cwl.input.json': {
 838                     'content': {
 839                         'x': 'XxX'
 840                     },
 841                     'kind': 'json'
 842                 }
 843             }, 'state': 'Committed',
 844             'output_path': '/var/spool/cwl',
 845             'name': 'expect_arvworkflow.cwl#main',
 846             'container_image': '999999999999999999999999999999d3+99',
 847             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 848                         '--no-log-timestamps', '--disable-validate',
 849                         '--eval-timeout=20', '--thread-count=1',
 850                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 851                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 852             'cwd': '/var/spool/cwl',
 853             'runtime_constraints': {
 854                 'API': True,
 855                 'vcpus': 1,
 856                 'ram': 1342177280
 857             },
 858             'use_existing': True,
 859             'properties': {},
 860             'secret_mounts': {}
 861         }
 862
 863         stubs.api.container_requests().create.assert_called_with(
 864             body=JsonDiffMatcher(expect_container))
 865         self.assertEqual(stubs.capture_stdout.getvalue(),
 866                          stubs.expect_container_request_uuid + '\n')
 867         self.assertEqual(exited, 0)
 868
 869     @mock.patch("arvados.collection.CollectionReader")
 870     @mock.patch("time.sleep")
 871     @stubs
 872     def test_submit_jobs_keepref(self, stubs, tm, reader):
 873         with open("tests/wf/expect_arvworkflow.cwl") as f:
 874             reader().open().__enter__().read.return_value = f.read()
 875
 876         exited = arvados_cwl.main(
 877             ["--submit", "--no-wait", "--api=jobs", "--debug",
 878              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 879             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 880
 881         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 882         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 883         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 884         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 885         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 886         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 887         stubs.api.pipeline_instances().create.assert_called_with(
 888             body=JsonDiffMatcher(expect_pipeline))
 889         self.assertEqual(exited, 0)
 890
 891     @mock.patch("time.sleep")
 892     @stubs
 893     def test_submit_arvworkflow(self, stubs, tm):
 894         with open("tests/wf/expect_arvworkflow.cwl") as f:
 895             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 896
 897         exited = arvados_cwl.main(
 898             ["--submit", "--no-wait", "--api=containers", "--debug",
 899              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 900             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 901
 902         expect_container = {
 903             'priority': 500,
 904             'mounts': {
 905                 '/var/spool/cwl': {
 906                     'writable': True,
 907                     'kind': 'collection'
 908                 },
 909                 'stdout': {
 910                     'path': '/var/spool/cwl/cwl.output.json',
 911                     'kind': 'file'
 912                 },
 913                 '/var/lib/cwl/workflow.json': {
 914                     'kind': 'json',
 915                     'content': {
 916                         'cwlVersion': 'v1.0',
 917                         '$graph': [
 918                             {
 919                                 'id': '#main',
 920                                 'inputs': [
 921                                     {'type': 'string', 'id': '#main/x'}
 922                                 ],
 923                                 'steps': [
 924                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 925                                      'run': '#submit_tool.cwl',
 926                                      'id': '#main/step1',
 927                                      'out': []}
 928                                 ],
 929                                 'class': 'Workflow',
 930                                 'outputs': []
 931                             },
 932                             {
 933                                 'inputs': [
 934                                     {
 935                                         'inputBinding': {'position': 1},
 936                                         'type': 'string',
 937                                         'id': '#submit_tool.cwl/x'}
 938                                 ],
 939                                 'requirements': [
 940                                     {
 941                                         'dockerPull': 'debian:8',
 942                                         'class': 'DockerRequirement',
 943                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 944                                     }
 945                                 ],
 946                                 'id': '#submit_tool.cwl',
 947                                 'outputs': [],
 948                                 'baseCommand': 'cat',
 949                                 'class': 'CommandLineTool'
 950                             }
 951                         ]
 952                     }
 953                 },
 954                 '/var/lib/cwl/cwl.input.json': {
 955                     'content': {
 956                         'x': 'XxX'
 957                     },
 958                     'kind': 'json'
 959                 }
 960             }, 'state': 'Committed',
 961             'output_path': '/var/spool/cwl',
 962             'name': 'a test workflow',
 963             'container_image': "999999999999999999999999999999d3+99",
 964             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 965                         '--no-log-timestamps', '--disable-validate',
 966                         '--eval-timeout=20', '--thread-count=1',
 967                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 968                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 969             'cwd': '/var/spool/cwl',
 970             'runtime_constraints': {
 971                 'API': True,
 972                 'vcpus': 1,
 973                 'ram': 1342177280
 974             },
 975             'use_existing': True,
 976             'properties': {
 977                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 978             },
 979             'secret_mounts': {}
 980         }
 981
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     @stubs
 989     def test_submit_container_name(self, stubs):
 990         exited = arvados_cwl.main(
 991             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 992                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 993             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 994
 995         expect_container = copy.deepcopy(stubs.expect_container_spec)
 996         expect_container["name"] = "hello container 123"
 997
 998         stubs.api.container_requests().create.assert_called_with(
 999             body=JsonDiffMatcher(expect_container))
1000         self.assertEqual(stubs.capture_stdout.getvalue(),
1001                          stubs.expect_container_request_uuid + '\n')
1002         self.assertEqual(exited, 0)
1003
1004     @stubs
1005     def test_submit_missing_input(self, stubs):
1006         exited = arvados_cwl.main(
1007             ["--submit", "--no-wait", "--api=containers", "--debug",
1008              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1009             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1010         self.assertEqual(exited, 0)
1011
1012         exited = arvados_cwl.main(
1013             ["--submit", "--no-wait", "--api=containers", "--debug",
1014              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
1015             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1016         self.assertEqual(exited, 1)
1017
1018     @stubs
1019     def test_submit_container_project(self, stubs):
1020         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1021         exited = arvados_cwl.main(
1022             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1023                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1024             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1025
1026         expect_container = copy.deepcopy(stubs.expect_container_spec)
1027         expect_container["owner_uuid"] = project_uuid
1028         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1029                                        '--no-log-timestamps', '--disable-validate',
1030                                        "--eval-timeout=20", "--thread-count=1",
1031                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
1032                                        '--on-error=continue',
1033                                        '--project-uuid='+project_uuid,
1034                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1035
1036         stubs.api.container_requests().create.assert_called_with(
1037             body=JsonDiffMatcher(expect_container))
1038         self.assertEqual(stubs.capture_stdout.getvalue(),
1039                          stubs.expect_container_request_uuid + '\n')
1040         self.assertEqual(exited, 0)
1041
1042     @stubs
1043     def test_submit_container_eval_timeout(self, stubs):
1044         exited = arvados_cwl.main(
1045             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1046                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1047             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1048
1049         expect_container = copy.deepcopy(stubs.expect_container_spec)
1050         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1051                                        '--no-log-timestamps', '--disable-validate',
1052                                        '--eval-timeout=60.0', '--thread-count=1',
1053                                        '--enable-reuse', "--collection-cache-size=256",
1054                                        '--debug', '--on-error=continue',
1055                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1056
1057         stubs.api.container_requests().create.assert_called_with(
1058             body=JsonDiffMatcher(expect_container))
1059         self.assertEqual(stubs.capture_stdout.getvalue(),
1060                          stubs.expect_container_request_uuid + '\n')
1061         self.assertEqual(exited, 0)
1062
1063     @stubs
1064     def test_submit_container_collection_cache(self, stubs):
1065         exited = arvados_cwl.main(
1066             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
1067                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1068             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1069
1070         expect_container = copy.deepcopy(stubs.expect_container_spec)
1071         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1072                                        '--no-log-timestamps', '--disable-validate',
1073                                        '--eval-timeout=20', '--thread-count=1',
1074                                        '--enable-reuse', "--collection-cache-size=500",
1075                                        '--debug', '--on-error=continue',
1076                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1077         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1078
1079         stubs.api.container_requests().create.assert_called_with(
1080             body=JsonDiffMatcher(expect_container))
1081         self.assertEqual(stubs.capture_stdout.getvalue(),
1082                          stubs.expect_container_request_uuid + '\n')
1083         self.assertEqual(exited, 0)
1084
1085     @stubs
1086     def test_submit_container_thread_count(self, stubs):
1087         exited = arvados_cwl.main(
1088             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1089                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1090             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1091
1092         expect_container = copy.deepcopy(stubs.expect_container_spec)
1093         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1094                                        '--no-log-timestamps', '--disable-validate',
1095                                        '--eval-timeout=20', '--thread-count=20',
1096                                        '--enable-reuse', "--collection-cache-size=256",
1097                                        '--debug', '--on-error=continue',
1098                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1099
1100         stubs.api.container_requests().create.assert_called_with(
1101             body=JsonDiffMatcher(expect_container))
1102         self.assertEqual(stubs.capture_stdout.getvalue(),
1103                          stubs.expect_container_request_uuid + '\n')
1104         self.assertEqual(exited, 0)
1105
1106     @stubs
1107     def test_submit_job_runner_image(self, stubs):
1108         exited = arvados_cwl.main(
1109             ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1110                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1111             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1112
1113         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
1114
1115         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1116         stubs.api.pipeline_instances().create.assert_called_with(
1117             body=JsonDiffMatcher(expect_pipeline))
1118         self.assertEqual(stubs.capture_stdout.getvalue(),
1119                          stubs.expect_pipeline_uuid + '\n')
1120         self.assertEqual(exited, 0)
1121
1122     @stubs
1123     def test_submit_container_runner_image(self, stubs):
1124         exited = arvados_cwl.main(
1125             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1126                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1127             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1128
1129         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1130
1131         expect_container = copy.deepcopy(stubs.expect_container_spec)
1132         stubs.api.container_requests().create.assert_called_with(
1133             body=JsonDiffMatcher(expect_container))
1134         self.assertEqual(stubs.capture_stdout.getvalue(),
1135                          stubs.expect_container_request_uuid + '\n')
1136         self.assertEqual(exited, 0)
1137
1138     @stubs
1139     def test_submit_priority(self, stubs):
1140         exited = arvados_cwl.main(
1141             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1142                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1143             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1144
1145         stubs.expect_container_spec["priority"] = 669
1146
1147         expect_container = copy.deepcopy(stubs.expect_container_spec)
1148         stubs.api.container_requests().create.assert_called_with(
1149             body=JsonDiffMatcher(expect_container))
1150         self.assertEqual(stubs.capture_stdout.getvalue(),
1151                          stubs.expect_container_request_uuid + '\n')
1152         self.assertEqual(exited, 0)
1153
1154     @stubs
1155     def test_submit_wf_runner_resources(self, stubs):
1156         exited = arvados_cwl.main(
1157             ["--submit", "--no-wait", "--api=containers", "--debug",
1158                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1159             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1160
1161         expect_container = copy.deepcopy(stubs.expect_container_spec)
1162         expect_container["runtime_constraints"] = {
1163             "API": True,
1164             "vcpus": 2,
1165             "ram": (2000+512) * 2**20
1166         }
1167         expect_container["name"] = "submit_wf_runner_resources.cwl"
1168         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1169             {
1170                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1171                 "coresMin": 2,
1172                 "ramMin": 2000,
1173                 "keep_cache": 512
1174             }
1175         ]
1176         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1177             "arv": "http://arvados.org/cwl#",
1178         }
1179         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1180                         '--no-log-timestamps', '--disable-validate',
1181                         '--eval-timeout=20', '--thread-count=1',
1182                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1183                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1184
1185         stubs.api.container_requests().create.assert_called_with(
1186             body=JsonDiffMatcher(expect_container))
1187         self.assertEqual(stubs.capture_stdout.getvalue(),
1188                          stubs.expect_container_request_uuid + '\n')
1189         self.assertEqual(exited, 0)
1190
1191     def tearDown(self):
1192         arvados_cwl.arvdocker.arv_docker_clear_cache()
1193
1194     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1195     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1196     @mock.patch("arvados.api")
1197     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1198         arvados_cwl.arvdocker.arv_docker_clear_cache()
1199
1200         arvrunner = mock.MagicMock()
1201         arvrunner.project_uuid = ""
1202         api.return_value = mock.MagicMock()
1203         arvrunner.api = api.return_value
1204         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1205                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1206                                                                         "link_class": "docker_image_repo+tag",
1207                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1208                                                                         "owner_uuid": "",
1209                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1210                                                             {"items": [{"created_at": "",
1211                                                                         "head_uuid": "",
1212                                                                         "link_class": "docker_image_hash",
1213                                                                         "name": "123456",
1214                                                                         "owner_uuid": "",
1215                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1216         )
1217         find_one_image_hash.return_value = "123456"
1218
1219         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1220                                                                               "owner_uuid": "",
1221                                                                               "manifest_text": "",
1222                                                                               "properties": ""
1223                                                                           }], "items_available": 1, "offset": 0},)
1224         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1225         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1226                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1227         self.assertEqual("9999999999999999999999999999999b+99",
1228                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1229
1230
1231     @stubs
1232     def test_submit_secrets(self, stubs):
1233         exited = arvados_cwl.main(
1234             ["--submit", "--no-wait", "--api=containers", "--debug",
1235                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1236             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1237
1238         expect_container = {
1239             "command": [
1240                 "arvados-cwl-runner",
1241                 "--local",
1242                 "--api=containers",
1243                 "--no-log-timestamps",
1244                 "--disable-validate",
1245                 "--eval-timeout=20",
1246                 '--thread-count=1',
1247                 "--enable-reuse",
1248                 "--collection-cache-size=256",
1249                 '--debug',
1250                 "--on-error=continue",
1251                 "/var/lib/cwl/workflow.json#main",
1252                 "/var/lib/cwl/cwl.input.json"
1253             ],
1254             "container_image": "999999999999999999999999999999d3+99",
1255             "cwd": "/var/spool/cwl",
1256             "mounts": {
1257                 "/var/lib/cwl/cwl.input.json": {
1258                     "content": {
1259                         "pw": {
1260                             "$include": "/secrets/s0"
1261                         }
1262                     },
1263                     "kind": "json"
1264                 },
1265                 "/var/lib/cwl/workflow.json": {
1266                     "content": {
1267                         "$graph": [
1268                             {
1269                                 "$namespaces": {
1270                                     "cwltool": "http://commonwl.org/cwltool#"
1271                                 },
1272                                 "arguments": [
1273                                     "md5sum",
1274                                     "example.conf"
1275                                 ],
1276                                 "class": "CommandLineTool",
1277                                 "hints": [
1278                                     {
1279                                         "class": "http://commonwl.org/cwltool#Secrets",
1280                                         "secrets": [
1281                                             "#secret_job.cwl/pw"
1282                                         ]
1283                                     }
1284                                 ],
1285                                 "id": "#secret_job.cwl",
1286                                 "inputs": [
1287                                     {
1288                                         "id": "#secret_job.cwl/pw",
1289                                         "type": "string"
1290                                     }
1291                                 ],
1292                                 "outputs": [
1293                                     {
1294                                         "id": "#secret_job.cwl/out",
1295                                         "type": "stdout"
1296                                     }
1297                                 ],
1298                                 "stdout": "hashed_example.txt",
1299                                 "requirements": [
1300                                     {
1301                                         "class": "InitialWorkDirRequirement",
1302                                         "listing": [
1303                                             {
1304                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1305                                                 "entryname": "example.conf"
1306                                             }
1307                                         ]
1308                                     }
1309                                 ]
1310                             },
1311                             {
1312                                 "class": "Workflow",
1313                                 "hints": [
1314                                     {
1315                                         "class": "DockerRequirement",
1316                                         "dockerPull": "debian:8",
1317                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1318                                     },
1319                                     {
1320                                         "class": "http://commonwl.org/cwltool#Secrets",
1321                                         "secrets": [
1322                                             "#main/pw"
1323                                         ]
1324                                     }
1325                                 ],
1326                                 "id": "#main",
1327                                 "inputs": [
1328                                     {
1329                                         "id": "#main/pw",
1330                                         "type": "string"
1331                                     }
1332                                 ],
1333                                 "outputs": [
1334                                     {
1335                                         "id": "#main/out",
1336                                         "outputSource": "#main/step1/out",
1337                                         "type": "File"
1338                                     }
1339                                 ],
1340                                 "steps": [
1341                                     {
1342                                         "id": "#main/step1",
1343                                         "in": [
1344                                             {
1345                                                 "id": "#main/step1/pw",
1346                                                 "source": "#main/pw"
1347                                             }
1348                                         ],
1349                                         "out": [
1350                                             "#main/step1/out"
1351                                         ],
1352                                         "run": "#secret_job.cwl"
1353                                     }
1354                                 ]
1355                             }
1356                         ],
1357                         "cwlVersion": "v1.0"
1358                     },
1359                     "kind": "json"
1360                 },
1361                 "/var/spool/cwl": {
1362                     "kind": "collection",
1363                     "writable": True
1364                 },
1365                 "stdout": {
1366                     "kind": "file",
1367                     "path": "/var/spool/cwl/cwl.output.json"
1368                 }
1369             },
1370             "name": "secret_wf.cwl",
1371             "output_path": "/var/spool/cwl",
1372             "priority": 500,
1373             "properties": {},
1374             "runtime_constraints": {
1375                 "API": True,
1376                 "ram": 1342177280,
1377                 "vcpus": 1
1378             },
1379             "secret_mounts": {
1380                 "/secrets/s0": {
1381                     "content": "blorp",
1382                     "kind": "text"
1383                 }
1384             },
1385             "state": "Committed",
1386             "use_existing": True
1387         }
1388
1389         stubs.api.container_requests().create.assert_called_with(
1390             body=JsonDiffMatcher(expect_container))
1391         self.assertEqual(stubs.capture_stdout.getvalue(),
1392                          stubs.expect_container_request_uuid + '\n')
1393         self.assertEqual(exited, 0)
1394
1395     @stubs
1396     def test_submit_request_uuid(self, stubs):
1397         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1398
1399         stubs.api.container_requests().update().execute.return_value = {
1400             "uuid": stubs.expect_container_request_uuid,
1401             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1402             "state": "Queued"
1403         }
1404
1405         exited = arvados_cwl.main(
1406             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1407                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1408             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1409
1410         stubs.api.container_requests().update.assert_called_with(
1411             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1412         self.assertEqual(stubs.capture_stdout.getvalue(),
1413                          stubs.expect_container_request_uuid + '\n')
1414         self.assertEqual(exited, 0)
1415
1416     @stubs
1417     def test_submit_container_cluster_id(self, stubs):
1418         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1419
1420         exited = arvados_cwl.main(
1421             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1422                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1423             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1424
1425         expect_container = copy.deepcopy(stubs.expect_container_spec)
1426
1427         stubs.api.container_requests().create.assert_called_with(
1428             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1429         self.assertEqual(stubs.capture_stdout.getvalue(),
1430                          stubs.expect_container_request_uuid + '\n')
1431         self.assertEqual(exited, 0)
1432
1433     @stubs
1434     def test_submit_validate_cluster_id(self, stubs):
1435         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1436         exited = arvados_cwl.main(
1437             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1438              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1439             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1440         self.assertEqual(exited, 1)
1441
1442     @mock.patch("arvados.collection.CollectionReader")
1443     @stubs
1444     def test_submit_uuid_inputs(self, stubs, collectionReader):
1445         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1446         def list_side_effect(**kwargs):
1447             m = mock.MagicMock()
1448             if "count" in kwargs:
1449                 m.execute.return_value = {"items": [
1450                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1451                 ]}
1452             else:
1453                 m.execute.return_value = {"items": []}
1454             return m
1455         stubs.api.collections().list.side_effect = list_side_effect
1456
1457         exited = arvados_cwl.main(
1458             ["--submit", "--no-wait", "--api=containers", "--debug",
1459                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1460             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1461
1462         expect_container = copy.deepcopy(stubs.expect_container_spec)
1463         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1464         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1465         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1466
1467         stubs.api.collections().list.assert_has_calls([
1468             mock.call(count='none',
1469                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1470                       select=['uuid', 'portable_data_hash'])])
1471         stubs.api.container_requests().create.assert_called_with(
1472             body=JsonDiffMatcher(expect_container))
1473         self.assertEqual(stubs.capture_stdout.getvalue(),
1474                          stubs.expect_container_request_uuid + '\n')
1475         self.assertEqual(exited, 0)
1476
1477     @stubs
1478     def test_submit_mismatched_uuid_inputs(self, stubs):
1479         def list_side_effect(**kwargs):
1480             m = mock.MagicMock()
1481             if "count" in kwargs:
1482                 m.execute.return_value = {"items": [
1483                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1484                 ]}
1485             else:
1486                 m.execute.return_value = {"items": []}
1487             return m
1488         stubs.api.collections().list.side_effect = list_side_effect
1489
1490         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1491             capture_stderr = StringIO()
1492             cwltool_logger = logging.getLogger('cwltool')
1493             stderr_logger = logging.StreamHandler(capture_stderr)
1494             cwltool_logger.addHandler(stderr_logger)
1495
1496             try:
1497                 exited = arvados_cwl.main(
1498                     ["--submit", "--no-wait", "--api=containers", "--debug",
1499                         "tests/wf/submit_wf.cwl", infile],
1500                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1501
1502                 self.assertEqual(exited, 1)
1503                 self.assertRegexpMatches(
1504                     capture_stderr.getvalue(),
1505                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1506             finally:
1507                 cwltool_logger.removeHandler(stderr_logger)
1508
1509     @mock.patch("arvados.collection.CollectionReader")
1510     @stubs
1511     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1512         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1513         capture_stderr = StringIO()
1514
1515         cwltool_logger = logging.getLogger('cwltool')
1516         stderr_logger = logging.StreamHandler(capture_stderr)
1517         cwltool_logger.addHandler(stderr_logger)
1518
1519         exited = arvados_cwl.main(
1520             ["--submit", "--no-wait", "--api=containers", "--debug",
1521                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1522             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1523
1524         try:
1525             self.assertEqual(exited, 1)
1526             self.assertRegexpMatches(
1527                 capture_stderr.getvalue(),
1528                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1529         finally:
1530             cwltool_logger.removeHandler(stderr_logger)
1531
1532
1533 class TestCreateTemplate(unittest.TestCase):
1534     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1535
1536     def _adjust_script_params(self, expect_component):
1537         expect_component['script_parameters']['x'] = {
1538             'dataclass': 'File',
1539             'required': True,
1540             'type': 'File',
1541             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1542         }
1543         expect_component['script_parameters']['y'] = {
1544             'dataclass': 'Collection',
1545             'required': True,
1546             'type': 'Directory',
1547             'value': '99999999999999999999999999999998+99',
1548         }
1549         expect_component['script_parameters']['z'] = {
1550             'dataclass': 'Collection',
1551             'required': True,
1552             'type': 'Directory',
1553         }
1554
1555     @stubs
1556     def test_create(self, stubs):
1557         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1558
1559         exited = arvados_cwl.main(
1560             ["--create-workflow", "--debug",
1561              "--api=jobs",
1562              "--project-uuid", project_uuid,
1563              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1564             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1565
1566         stubs.api.pipeline_instances().create.refute_called()
1567         stubs.api.jobs().create.refute_called()
1568
1569         expect_component = copy.deepcopy(stubs.expect_job_spec)
1570         self._adjust_script_params(expect_component)
1571         expect_template = {
1572             "components": {
1573                 "submit_wf.cwl": expect_component,
1574             },
1575             "name": "submit_wf.cwl",
1576             "owner_uuid": project_uuid,
1577         }
1578         stubs.api.pipeline_templates().create.assert_called_with(
1579             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1580
1581         self.assertEqual(stubs.capture_stdout.getvalue(),
1582                          stubs.expect_pipeline_template_uuid + '\n')
1583         self.assertEqual(exited, 0)
1584
1585     @stubs
1586     def test_create_name(self, stubs):
1587         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1588
1589         exited = arvados_cwl.main(
1590             ["--create-workflow", "--debug",
1591              "--project-uuid", project_uuid,
1592              "--api=jobs",
1593              "--name", "testing 123",
1594              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1595             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1596
1597         stubs.api.pipeline_instances().create.refute_called()
1598         stubs.api.jobs().create.refute_called()
1599
1600         expect_component = copy.deepcopy(stubs.expect_job_spec)
1601         self._adjust_script_params(expect_component)
1602         expect_template = {
1603             "components": {
1604                 "testing 123": expect_component,
1605             },
1606             "name": "testing 123",
1607             "owner_uuid": project_uuid,
1608         }
1609         stubs.api.pipeline_templates().create.assert_called_with(
1610             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1611
1612         self.assertEqual(stubs.capture_stdout.getvalue(),
1613                          stubs.expect_pipeline_template_uuid + '\n')
1614         self.assertEqual(exited, 0)
1615
1616     @stubs
1617     def test_update_name(self, stubs):
1618         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1619
1620         exited = arvados_cwl.main(
1621             ["--update-workflow", self.existing_template_uuid,
1622              "--debug",
1623              "--project-uuid", project_uuid,
1624              "--api=jobs",
1625              "--name", "testing 123",
1626              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1627             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1628
1629         stubs.api.pipeline_instances().create.refute_called()
1630         stubs.api.jobs().create.refute_called()
1631
1632         expect_component = copy.deepcopy(stubs.expect_job_spec)
1633         self._adjust_script_params(expect_component)
1634         expect_template = {
1635             "components": {
1636                 "testing 123": expect_component,
1637             },
1638             "name": "testing 123",
1639             "owner_uuid": project_uuid,
1640         }
1641         stubs.api.pipeline_templates().create.refute_called()
1642         stubs.api.pipeline_templates().update.assert_called_with(
1643             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1644
1645         self.assertEqual(stubs.capture_stdout.getvalue(),
1646                          self.existing_template_uuid + '\n')
1647         self.assertEqual(exited, 0)
1648
1649
1650 class TestCreateWorkflow(unittest.TestCase):
1651     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1652     expect_workflow = StripYAMLComments(
1653         open("tests/wf/expect_packed.cwl").read())
1654
1655     @stubs
1656     def test_create(self, stubs):
1657         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1658
1659         exited = arvados_cwl.main(
1660             ["--create-workflow", "--debug",
1661              "--api=containers",
1662              "--project-uuid", project_uuid,
1663              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1664             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1665
1666         stubs.api.pipeline_templates().create.refute_called()
1667         stubs.api.container_requests().create.refute_called()
1668
1669         body = {
1670             "workflow": {
1671                 "owner_uuid": project_uuid,
1672                 "name": "submit_wf.cwl",
1673                 "description": "",
1674                 "definition": self.expect_workflow,
1675             }
1676         }
1677         stubs.api.workflows().create.assert_called_with(
1678             body=JsonDiffMatcher(body))
1679
1680         self.assertEqual(stubs.capture_stdout.getvalue(),
1681                          stubs.expect_workflow_uuid + '\n')
1682         self.assertEqual(exited, 0)
1683
1684     @stubs
1685     def test_create_name(self, stubs):
1686         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1687
1688         exited = arvados_cwl.main(
1689             ["--create-workflow", "--debug",
1690              "--api=containers",
1691              "--project-uuid", project_uuid,
1692              "--name", "testing 123",
1693              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1694             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1695
1696         stubs.api.pipeline_templates().create.refute_called()
1697         stubs.api.container_requests().create.refute_called()
1698
1699         body = {
1700             "workflow": {
1701                 "owner_uuid": project_uuid,
1702                 "name": "testing 123",
1703                 "description": "",
1704                 "definition": self.expect_workflow,
1705             }
1706         }
1707         stubs.api.workflows().create.assert_called_with(
1708             body=JsonDiffMatcher(body))
1709
1710         self.assertEqual(stubs.capture_stdout.getvalue(),
1711                          stubs.expect_workflow_uuid + '\n')
1712         self.assertEqual(exited, 0)
1713
1714     @stubs
1715     def test_incompatible_api(self, stubs):
1716         capture_stderr = StringIO()
1717         acr_logger = logging.getLogger('arvados.cwl-runner')
1718         stderr_logger = logging.StreamHandler(capture_stderr)
1719         acr_logger.addHandler(stderr_logger)
1720
1721         try:
1722             exited = arvados_cwl.main(
1723                 ["--update-workflow", self.existing_workflow_uuid,
1724                  "--api=jobs",
1725                  "--debug",
1726                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1727                 sys.stderr, sys.stderr, api_client=stubs.api)
1728             self.assertEqual(exited, 1)
1729             self.assertRegexpMatches(
1730                 capture_stderr.getvalue(),
1731                 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1732         finally:
1733             acr_logger.removeHandler(stderr_logger)
1734
1735     @stubs
1736     def test_update(self, stubs):
1737         exited = arvados_cwl.main(
1738             ["--update-workflow", self.existing_workflow_uuid,
1739              "--debug",
1740              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1741             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1742
1743         body = {
1744             "workflow": {
1745                 "name": "submit_wf.cwl",
1746                 "description": "",
1747                 "definition": self.expect_workflow,
1748             }
1749         }
1750         stubs.api.workflows().update.assert_called_with(
1751             uuid=self.existing_workflow_uuid,
1752             body=JsonDiffMatcher(body))
1753         self.assertEqual(stubs.capture_stdout.getvalue(),
1754                          self.existing_workflow_uuid + '\n')
1755         self.assertEqual(exited, 0)
1756
1757     @stubs
1758     def test_update_name(self, stubs):
1759         exited = arvados_cwl.main(
1760             ["--update-workflow", self.existing_workflow_uuid,
1761              "--debug", "--name", "testing 123",
1762              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1763             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1764
1765         body = {
1766             "workflow": {
1767                 "name": "testing 123",
1768                 "description": "",
1769                 "definition": self.expect_workflow,
1770             }
1771         }
1772         stubs.api.workflows().update.assert_called_with(
1773             uuid=self.existing_workflow_uuid,
1774             body=JsonDiffMatcher(body))
1775         self.assertEqual(stubs.capture_stdout.getvalue(),
1776                          self.existing_workflow_uuid + '\n')
1777         self.assertEqual(exited, 0)
1778
1779     @stubs
1780     def test_create_collection_per_tool(self, stubs):
1781         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1782
1783         exited = arvados_cwl.main(
1784             ["--create-workflow", "--debug",
1785              "--api=containers",
1786              "--project-uuid", project_uuid,
1787              "tests/collection_per_tool/collection_per_tool.cwl"],
1788             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1789
1790         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1791         expect_workflow = StripYAMLComments(open(toolfile).read())
1792
1793         body = {
1794             "workflow": {
1795                 "owner_uuid": project_uuid,
1796                 "name": "collection_per_tool.cwl",
1797                 "description": "",
1798                 "definition": expect_workflow,
1799             }
1800         }
1801         stubs.api.workflows().create.assert_called_with(
1802             body=JsonDiffMatcher(body))
1803
1804         self.assertEqual(stubs.capture_stdout.getvalue(),
1805                          stubs.expect_workflow_uuid + '\n')
1806         self.assertEqual(exited, 0)
1807
1808 class TestTemplateInputs(unittest.TestCase):
1809     expect_template = {
1810         "components": {
1811             "inputs_test.cwl": {
1812                 'runtime_constraints': {
1813                     'docker_image': '999999999999999999999999999999d3+99',
1814                     'min_ram_mb_per_node': 1024
1815                 },
1816                 'script_parameters': {
1817                     'cwl:tool':
1818                     'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
1819                     'optionalFloatInput': None,
1820                     'fileInput': {
1821                         'type': 'File',
1822                         'dataclass': 'File',
1823                         'required': True,
1824                         'title': "It's a file; we expect to find some characters in it.",
1825                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1826                     },
1827                     'floatInput': {
1828                         'type': 'float',
1829                         'dataclass': 'number',
1830                         'required': True,
1831                         'title': 'Floats like a duck',
1832                         'default': 0.1,
1833                         'value': 0.1,
1834                     },
1835                     'optionalFloatInput': {
1836                         'type': ['null', 'float'],
1837                         'dataclass': 'number',
1838                         'required': False,
1839                     },
1840                     'boolInput': {
1841                         'type': 'boolean',
1842                         'dataclass': 'boolean',
1843                         'required': True,
1844                         'title': 'True or false?',
1845                     },
1846                 },
1847                 'repository': 'arvados',
1848                 'script_version': 'master',
1849                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1850                 'script': 'cwl-runner',
1851             },
1852         },
1853         "name": "inputs_test.cwl",
1854     }
1855
1856     @stubs
1857     def test_inputs_empty(self, stubs):
1858         exited = arvados_cwl.main(
1859             ["--create-template",
1860              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1861             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1862
1863         stubs.api.pipeline_templates().create.assert_called_with(
1864             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1865
1866         self.assertEqual(exited, 0)
1867
1868     @stubs
1869     def test_inputs(self, stubs):
1870         exited = arvados_cwl.main(
1871             ["--create-template",
1872              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1873             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1874
1875         expect_template = copy.deepcopy(self.expect_template)
1876         params = expect_template[
1877             "components"]["inputs_test.cwl"]["script_parameters"]
1878         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1879         params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
1880         params["floatInput"]["value"] = 1.234
1881         params["boolInput"]["value"] = True
1882
1883         stubs.api.pipeline_templates().create.assert_called_with(
1884             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1885         self.assertEqual(exited, 0)