sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 1,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'state': 'Committed',
 235             'owner_uuid': None,
 236             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 237                         '--enable-reuse', '--on-error=continue',
 238                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 239             'name': 'submit_wf.cwl',
 240             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 241             'output_path': '/var/spool/cwl',
 242             'cwd': '/var/spool/cwl',
 243             'runtime_constraints': {
 244                 'API': True,
 245                 'vcpus': 1,
 246                 'ram': 1024*1024*1024
 247             },
 248             'use_existing': True,
 249             'properties': {}
 250         }
 251
 252         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 253         stubs.api.workflows().create().execute.return_value = {
 254             "uuid": stubs.expect_workflow_uuid,
 255         }
 256         def update_mock(**kwargs):
 257             stubs.updated_uuid = kwargs.get('uuid')
 258             return mock.DEFAULT
 259         stubs.api.workflows().update.side_effect = update_mock
 260         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 261             "uuid": stubs.updated_uuid,
 262         }
 263
 264         return func(self, stubs, *args, **kwargs)
 265     return wrapped
 266
 267
 268 class TestSubmit(unittest.TestCase):
 269     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit(self, stubs, tm, arvdock):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--api=jobs", "--debug",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.api.collections().create.assert_has_calls([
 281             mock.call(body=JsonDiffMatcher({
 282                 'manifest_text':
 283                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 284                 'replication_desired': None,
 285                 'name': 'submit_tool.cwl dependencies',
 286             }), ensure_unique_name=True),
 287             mock.call(body=JsonDiffMatcher({
 288                 'manifest_text':
 289                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 290                 'replication_desired': None,
 291                 'name': 'submit_wf.cwl input',
 292             }), ensure_unique_name=True),
 293             mock.call(body=JsonDiffMatcher({
 294                 'manifest_text':
 295                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 296                 'replication_desired': None,
 297                 'name': 'submit_wf.cwl',
 298             }), ensure_unique_name=True)        ])
 299
 300         arvdock.assert_has_calls([
 301             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 302             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 303         ])
 304
 305         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 306         stubs.api.pipeline_instances().create.assert_called_with(
 307             body=JsonDiffMatcher(expect_pipeline))
 308         self.assertEqual(capture_stdout.getvalue(),
 309                          stubs.expect_pipeline_uuid + '\n')
 310
 311
 312     @mock.patch("time.sleep")
 313     @stubs
 314     def test_submit_no_reuse(self, stubs, tm):
 315         capture_stdout = cStringIO.StringIO()
 316         exited = arvados_cwl.main(
 317             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 318              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 319             capture_stdout, sys.stderr, api_client=stubs.api)
 320         self.assertEqual(exited, 0)
 321
 322         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 323         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 324
 325         stubs.api.pipeline_instances().create.assert_called_with(
 326             body=JsonDiffMatcher(expect_pipeline))
 327         self.assertEqual(capture_stdout.getvalue(),
 328                          stubs.expect_pipeline_uuid + '\n')
 329
 330     @mock.patch("time.sleep")
 331     @stubs
 332     def test_submit_on_error(self, stubs, tm):
 333         capture_stdout = cStringIO.StringIO()
 334         exited = arvados_cwl.main(
 335             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 336              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 337             capture_stdout, sys.stderr, api_client=stubs.api)
 338         self.assertEqual(exited, 0)
 339
 340         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 341         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 342
 343         stubs.api.pipeline_instances().create.assert_called_with(
 344             body=JsonDiffMatcher(expect_pipeline))
 345         self.assertEqual(capture_stdout.getvalue(),
 346                          stubs.expect_pipeline_uuid + '\n')
 347
 348
 349     @mock.patch("time.sleep")
 350     @stubs
 351     def test_submit_runner_ram(self, stubs, tm):
 352         capture_stdout = cStringIO.StringIO()
 353         exited = arvados_cwl.main(
 354             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 355              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 356             capture_stdout, sys.stderr, api_client=stubs.api)
 357         self.assertEqual(exited, 0)
 358
 359         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 360         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 361
 362         stubs.api.pipeline_instances().create.assert_called_with(
 363             body=JsonDiffMatcher(expect_pipeline))
 364         self.assertEqual(capture_stdout.getvalue(),
 365                          stubs.expect_pipeline_uuid + '\n')
 366
 367
 368     @mock.patch("time.sleep")
 369     @stubs
 370     def test_submit_invalid_runner_ram(self, stubs, tm):
 371         capture_stdout = cStringIO.StringIO()
 372         exited = arvados_cwl.main(
 373             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 374              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 375             capture_stdout, sys.stderr, api_client=stubs.api)
 376         self.assertEqual(exited, 1)
 377
 378     @mock.patch("time.sleep")
 379     @stubs
 380     def test_submit_output_name(self, stubs, tm):
 381         output_name = "test_output_name"
 382
 383         capture_stdout = cStringIO.StringIO()
 384         exited = arvados_cwl.main(
 385             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 386              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 387             capture_stdout, sys.stderr, api_client=stubs.api)
 388         self.assertEqual(exited, 0)
 389
 390         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 391         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 392
 393         stubs.api.pipeline_instances().create.assert_called_with(
 394             body=JsonDiffMatcher(expect_pipeline))
 395         self.assertEqual(capture_stdout.getvalue(),
 396                          stubs.expect_pipeline_uuid + '\n')
 397
 398
 399     @mock.patch("time.sleep")
 400     @stubs
 401     def test_submit_pipeline_name(self, stubs, tm):
 402         capture_stdout = cStringIO.StringIO()
 403         exited = arvados_cwl.main(
 404             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 405              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 406             capture_stdout, sys.stderr, api_client=stubs.api)
 407         self.assertEqual(exited, 0)
 408
 409         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 410         expect_pipeline["name"] = "hello job 123"
 411
 412         stubs.api.pipeline_instances().create.assert_called_with(
 413             body=JsonDiffMatcher(expect_pipeline))
 414         self.assertEqual(capture_stdout.getvalue(),
 415                          stubs.expect_pipeline_uuid + '\n')
 416
 417     @mock.patch("time.sleep")
 418     @stubs
 419     def test_submit_output_tags(self, stubs, tm):
 420         output_tags = "tag0,tag1,tag2"
 421
 422         capture_stdout = cStringIO.StringIO()
 423         exited = arvados_cwl.main(
 424             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 425              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 426             capture_stdout, sys.stderr, api_client=stubs.api)
 427         self.assertEqual(exited, 0)
 428
 429         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 430         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 431
 432         stubs.api.pipeline_instances().create.assert_called_with(
 433             body=JsonDiffMatcher(expect_pipeline))
 434         self.assertEqual(capture_stdout.getvalue(),
 435                          stubs.expect_pipeline_uuid + '\n')
 436
 437     @mock.patch("time.sleep")
 438     @stubs
 439     def test_submit_with_project_uuid(self, stubs, tm):
 440         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 441
 442         exited = arvados_cwl.main(
 443             ["--submit", "--no-wait",
 444              "--project-uuid", project_uuid,
 445              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 446             sys.stdout, sys.stderr, api_client=stubs.api)
 447         self.assertEqual(exited, 0)
 448
 449         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 450         expect_pipeline["owner_uuid"] = project_uuid
 451         stubs.api.pipeline_instances().create.assert_called_with(
 452             body=JsonDiffMatcher(expect_pipeline))
 453
 454     @stubs
 455     def test_submit_container(self, stubs):
 456         capture_stdout = cStringIO.StringIO()
 457         try:
 458             exited = arvados_cwl.main(
 459                 ["--submit", "--no-wait", "--api=containers", "--debug",
 460                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 461                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 462             self.assertEqual(exited, 0)
 463         except:
 464             logging.exception("")
 465
 466         stubs.api.collections().create.assert_has_calls([
 467             mock.call(body=JsonDiffMatcher({
 468                 'manifest_text':
 469                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 470                 'replication_desired': None,
 471                 'name': 'submit_tool.cwl dependencies',
 472             }), ensure_unique_name=True),
 473             mock.call(body=JsonDiffMatcher({
 474                 'manifest_text':
 475                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 476                 'replication_desired': None,
 477                 'name': 'submit_wf.cwl input',
 478             }), ensure_unique_name=True)])
 479
 480         expect_container = copy.deepcopy(stubs.expect_container_spec)
 481         stubs.api.container_requests().create.assert_called_with(
 482             body=JsonDiffMatcher(expect_container))
 483         self.assertEqual(capture_stdout.getvalue(),
 484                          stubs.expect_container_request_uuid + '\n')
 485
 486     @stubs
 487     def test_submit_container_no_reuse(self, stubs):
 488         capture_stdout = cStringIO.StringIO()
 489         try:
 490             exited = arvados_cwl.main(
 491                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 492                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 493                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 494             self.assertEqual(exited, 0)
 495         except:
 496             logging.exception("")
 497
 498         expect_container = copy.deepcopy(stubs.expect_container_spec)
 499         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 500                                                   '--disable-reuse', '--on-error=continue',
 501                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 502         expect_container["use_existing"] = False
 503
 504         stubs.api.container_requests().create.assert_called_with(
 505             body=JsonDiffMatcher(expect_container))
 506         self.assertEqual(capture_stdout.getvalue(),
 507                          stubs.expect_container_request_uuid + '\n')
 508
 509
 510     @stubs
 511     def test_submit_container_on_error(self, stubs):
 512         capture_stdout = cStringIO.StringIO()
 513         try:
 514             exited = arvados_cwl.main(
 515                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 516                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 517                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 518             self.assertEqual(exited, 0)
 519         except:
 520             logging.exception("")
 521
 522         expect_container = copy.deepcopy(stubs.expect_container_spec)
 523         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 524                                                   '--enable-reuse', '--on-error=stop',
 525                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 526
 527         stubs.api.container_requests().create.assert_called_with(
 528             body=JsonDiffMatcher(expect_container))
 529         self.assertEqual(capture_stdout.getvalue(),
 530                          stubs.expect_container_request_uuid + '\n')
 531
 532     @stubs
 533     def test_submit_container_output_name(self, stubs):
 534         output_name = "test_output_name"
 535
 536         capture_stdout = cStringIO.StringIO()
 537         try:
 538             exited = arvados_cwl.main(
 539                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 540                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 541                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 542             self.assertEqual(exited, 0)
 543         except:
 544             logging.exception("")
 545
 546         expect_container = copy.deepcopy(stubs.expect_container_spec)
 547         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 548                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 549                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 550         expect_container["output_name"] = output_name
 551
 552         stubs.api.container_requests().create.assert_called_with(
 553             body=JsonDiffMatcher(expect_container))
 554         self.assertEqual(capture_stdout.getvalue(),
 555                          stubs.expect_container_request_uuid + '\n')
 556
 557
 558     @stubs
 559     def test_submit_container_output_ttl(self, stubs):
 560         capture_stdout = cStringIO.StringIO()
 561         try:
 562             exited = arvados_cwl.main(
 563                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 564                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 565                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 566             self.assertEqual(exited, 0)
 567         except:
 568             logging.exception("")
 569
 570         expect_container = copy.deepcopy(stubs.expect_container_spec)
 571         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 572                                        '--enable-reuse', '--on-error=continue',
 573                                        "--intermediate-output-ttl=3600",
 574                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 575
 576         stubs.api.container_requests().create.assert_called_with(
 577             body=JsonDiffMatcher(expect_container))
 578         self.assertEqual(capture_stdout.getvalue(),
 579                          stubs.expect_container_request_uuid + '\n')
 580
 581     @stubs
 582     def test_submit_container_trash_intermediate(self, stubs):
 583         capture_stdout = cStringIO.StringIO()
 584         try:
 585             exited = arvados_cwl.main(
 586                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 587                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 588                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 589             self.assertEqual(exited, 0)
 590         except:
 591             logging.exception("")
 592
 593         expect_container = copy.deepcopy(stubs.expect_container_spec)
 594         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 595                                        '--enable-reuse', '--on-error=continue',
 596                                        "--trash-intermediate",
 597                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 598
 599         stubs.api.container_requests().create.assert_called_with(
 600             body=JsonDiffMatcher(expect_container))
 601         self.assertEqual(capture_stdout.getvalue(),
 602                          stubs.expect_container_request_uuid + '\n')
 603
 604     @stubs
 605     def test_submit_container_output_tags(self, stubs):
 606         output_tags = "tag0,tag1,tag2"
 607
 608         capture_stdout = cStringIO.StringIO()
 609         try:
 610             exited = arvados_cwl.main(
 611                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 612                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 613                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 614             self.assertEqual(exited, 0)
 615         except:
 616             logging.exception("")
 617
 618         expect_container = copy.deepcopy(stubs.expect_container_spec)
 619         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 620                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 621                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 622
 623         stubs.api.container_requests().create.assert_called_with(
 624             body=JsonDiffMatcher(expect_container))
 625         self.assertEqual(capture_stdout.getvalue(),
 626                          stubs.expect_container_request_uuid + '\n')
 627
 628     @stubs
 629     def test_submit_container_runner_ram(self, stubs):
 630         capture_stdout = cStringIO.StringIO()
 631         try:
 632             exited = arvados_cwl.main(
 633                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 634                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 635                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 636             self.assertEqual(exited, 0)
 637         except:
 638             logging.exception("")
 639
 640         expect_container = copy.deepcopy(stubs.expect_container_spec)
 641         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 642
 643         stubs.api.container_requests().create.assert_called_with(
 644             body=JsonDiffMatcher(expect_container))
 645         self.assertEqual(capture_stdout.getvalue(),
 646                          stubs.expect_container_request_uuid + '\n')
 647
 648     @mock.patch("arvados.collection.CollectionReader")
 649     @mock.patch("time.sleep")
 650     @stubs
 651     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 652         capture_stdout = cStringIO.StringIO()
 653         exited = arvados_cwl.main(
 654             ["--submit", "--no-wait", "--api=containers", "--debug",
 655              "tests/wf/submit_keepref_wf.cwl"],
 656             capture_stdout, sys.stderr, api_client=stubs.api)
 657         self.assertEqual(exited, 0)
 658
 659
 660     @mock.patch("arvados.collection.CollectionReader")
 661     @mock.patch("time.sleep")
 662     @stubs
 663     def test_submit_keepref(self, stubs, tm, reader):
 664         capture_stdout = cStringIO.StringIO()
 665
 666         with open("tests/wf/expect_arvworkflow.cwl") as f:
 667             reader().open().__enter__().read.return_value = f.read()
 668
 669         exited = arvados_cwl.main(
 670             ["--submit", "--no-wait", "--api=containers", "--debug",
 671              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 672             capture_stdout, sys.stderr, api_client=stubs.api)
 673         self.assertEqual(exited, 0)
 674
 675         expect_container = {
 676             'priority': 1,
 677             'mounts': {
 678                 '/var/spool/cwl': {
 679                     'writable': True,
 680                     'kind': 'collection'
 681                 },
 682                 'stdout': {
 683                     'path': '/var/spool/cwl/cwl.output.json',
 684                     'kind': 'file'
 685                 },
 686                 '/var/lib/cwl/workflow': {
 687                     'portable_data_hash': '99999999999999999999999999999994+99',
 688                     'kind': 'collection'
 689                 },
 690                 '/var/lib/cwl/cwl.input.json': {
 691                     'content': {
 692                         'x': 'XxX'
 693                     },
 694                     'kind': 'json'
 695                 }
 696             }, 'state': 'Committed',
 697             'owner_uuid': None,
 698             'output_path': '/var/spool/cwl',
 699             'name': 'expect_arvworkflow.cwl#main',
 700             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 701             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 702                         '--enable-reuse', '--on-error=continue',
 703                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 704             'cwd': '/var/spool/cwl',
 705             'runtime_constraints': {
 706                 'API': True,
 707                 'vcpus': 1,
 708                 'ram': 1073741824
 709             },
 710             'use_existing': True,
 711             'properties': {}
 712         }
 713
 714         stubs.api.container_requests().create.assert_called_with(
 715             body=JsonDiffMatcher(expect_container))
 716         self.assertEqual(capture_stdout.getvalue(),
 717                          stubs.expect_container_request_uuid + '\n')
 718
 719
 720     @mock.patch("arvados.collection.CollectionReader")
 721     @mock.patch("time.sleep")
 722     @stubs
 723     def test_submit_jobs_keepref(self, stubs, tm, reader):
 724         capture_stdout = cStringIO.StringIO()
 725
 726         with open("tests/wf/expect_arvworkflow.cwl") as f:
 727             reader().open().__enter__().read.return_value = f.read()
 728
 729         exited = arvados_cwl.main(
 730             ["--submit", "--no-wait", "--api=jobs", "--debug",
 731              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 732             capture_stdout, sys.stderr, api_client=stubs.api)
 733         self.assertEqual(exited, 0)
 734
 735         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 736         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 737         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 738         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 739         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 740         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 741         stubs.api.pipeline_instances().create.assert_called_with(
 742             body=JsonDiffMatcher(expect_pipeline))
 743
 744     @mock.patch("time.sleep")
 745     @stubs
 746     def test_submit_arvworkflow(self, stubs, tm):
 747         capture_stdout = cStringIO.StringIO()
 748
 749         with open("tests/wf/expect_arvworkflow.cwl") as f:
 750             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 751
 752         exited = arvados_cwl.main(
 753             ["--submit", "--no-wait", "--api=containers", "--debug",
 754              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 755             capture_stdout, sys.stderr, api_client=stubs.api)
 756         self.assertEqual(exited, 0)
 757
 758         expect_container = {
 759             'priority': 1,
 760             'mounts': {
 761                 '/var/spool/cwl': {
 762                     'writable': True,
 763                     'kind': 'collection'
 764                 },
 765                 'stdout': {
 766                     'path': '/var/spool/cwl/cwl.output.json',
 767                     'kind': 'file'
 768                 },
 769                 '/var/lib/cwl/workflow.json': {
 770                     'kind': 'json',
 771                     'content': {
 772                         'cwlVersion': 'v1.0',
 773                         '$graph': [
 774                             {
 775                                 'id': '#main',
 776                                 'inputs': [
 777                                     {'type': 'string', 'id': '#main/x'}
 778                                 ],
 779                                 'steps': [
 780                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 781                                      'run': '#submit_tool.cwl',
 782                                      'id': '#main/step1',
 783                                      'out': []}
 784                                 ],
 785                                 'class': 'Workflow',
 786                                 'outputs': []
 787                             },
 788                             {
 789                                 'inputs': [
 790                                     {
 791                                         'inputBinding': {'position': 1},
 792                                         'type': 'string',
 793                                         'id': '#submit_tool.cwl/x'}
 794                                 ],
 795                                 'requirements': [
 796                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 797                                 ],
 798                                 'id': '#submit_tool.cwl',
 799                                 'outputs': [],
 800                                 'baseCommand': 'cat',
 801                                 'class': 'CommandLineTool'
 802                             }
 803                         ]
 804                     }
 805                 },
 806                 '/var/lib/cwl/cwl.input.json': {
 807                     'content': {
 808                         'x': 'XxX'
 809                     },
 810                     'kind': 'json'
 811                 }
 812             }, 'state': 'Committed',
 813             'owner_uuid': None,
 814             'output_path': '/var/spool/cwl',
 815             'name': 'a test workflow',
 816             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 817             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 818                         '--enable-reuse', '--on-error=continue',
 819                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 820             'cwd': '/var/spool/cwl',
 821             'runtime_constraints': {
 822                 'API': True,
 823                 'vcpus': 1,
 824                 'ram': 1073741824
 825             },
 826             'use_existing': True,
 827             'properties': {
 828                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 829             }
 830         }
 831
 832         stubs.api.container_requests().create.assert_called_with(
 833             body=JsonDiffMatcher(expect_container))
 834         self.assertEqual(capture_stdout.getvalue(),
 835                          stubs.expect_container_request_uuid + '\n')
 836
 837
 838     @stubs
 839     def test_submit_container_name(self, stubs):
 840         capture_stdout = cStringIO.StringIO()
 841         try:
 842             exited = arvados_cwl.main(
 843                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 844                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 845                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 846             self.assertEqual(exited, 0)
 847         except:
 848             logging.exception("")
 849
 850         expect_container = copy.deepcopy(stubs.expect_container_spec)
 851         expect_container["name"] = "hello container 123"
 852
 853         stubs.api.container_requests().create.assert_called_with(
 854             body=JsonDiffMatcher(expect_container))
 855         self.assertEqual(capture_stdout.getvalue(),
 856                          stubs.expect_container_request_uuid + '\n')
 857
 858
 859     @stubs
 860     def test_submit_container_project(self, stubs):
 861         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 862         capture_stdout = cStringIO.StringIO()
 863         try:
 864             exited = arvados_cwl.main(
 865                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 866                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 867                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 868             self.assertEqual(exited, 0)
 869         except:
 870             logging.exception("")
 871
 872         expect_container = copy.deepcopy(stubs.expect_container_spec)
 873         expect_container["owner_uuid"] = project_uuid
 874         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 875                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid,
 876                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 877
 878         stubs.api.container_requests().create.assert_called_with(
 879             body=JsonDiffMatcher(expect_container))
 880         self.assertEqual(capture_stdout.getvalue(),
 881                          stubs.expect_container_request_uuid + '\n')
 882
 883
 884     @stubs
 885     def test_submit_job_runner_image(self, stubs):
 886         capture_stdout = cStringIO.StringIO()
 887         try:
 888             exited = arvados_cwl.main(
 889                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 890                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 891                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 892             self.assertEqual(exited, 0)
 893         except:
 894             logging.exception("")
 895
 896         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 897
 898         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 899         stubs.api.pipeline_instances().create.assert_called_with(
 900             body=JsonDiffMatcher(expect_pipeline))
 901         self.assertEqual(capture_stdout.getvalue(),
 902                          stubs.expect_pipeline_uuid + '\n')
 903
 904     @stubs
 905     def test_submit_container_runner_image(self, stubs):
 906         capture_stdout = cStringIO.StringIO()
 907         try:
 908             exited = arvados_cwl.main(
 909                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 910                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 911                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 912             self.assertEqual(exited, 0)
 913         except:
 914             logging.exception("")
 915
 916         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 917
 918         expect_container = copy.deepcopy(stubs.expect_container_spec)
 919         stubs.api.container_requests().create.assert_called_with(
 920             body=JsonDiffMatcher(expect_container))
 921         self.assertEqual(capture_stdout.getvalue(),
 922                          stubs.expect_container_request_uuid + '\n')
 923
 924
 925     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 926     @mock.patch("cwltool.docker.get_image")
 927     @mock.patch("arvados.api")
 928     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 929         arvrunner = mock.MagicMock()
 930         arvrunner.project_uuid = ""
 931         api.return_value = mock.MagicMock()
 932         arvrunner.api = api.return_value
 933         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 934                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 935                                                                         "link_class": "docker_image_repo+tag",
 936                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 937                                                                         "owner_uuid": "",
 938                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 939                                                             {"items": [{"created_at": "",
 940                                                                         "head_uuid": "",
 941                                                                         "link_class": "docker_image_hash",
 942                                                                         "name": "123456",
 943                                                                         "owner_uuid": "",
 944                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 945         )
 946         find_one_image_hash.return_value = "123456"
 947
 948         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 949                                                                               "owner_uuid": "",
 950                                                                               "manifest_text": "",
 951                                                                               "properties": ""
 952                                                                           }], "items_available": 1, "offset": 0},)
 953         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 954         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 955                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 956
 957 class TestCreateTemplate(unittest.TestCase):
 958     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 959
 960     def _adjust_script_params(self, expect_component):
 961         expect_component['script_parameters']['x'] = {
 962             'dataclass': 'File',
 963             'required': True,
 964             'type': 'File',
 965             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 966         }
 967         expect_component['script_parameters']['y'] = {
 968             'dataclass': 'Collection',
 969             'required': True,
 970             'type': 'Directory',
 971             'value': '99999999999999999999999999999998+99',
 972         }
 973         expect_component['script_parameters']['z'] = {
 974             'dataclass': 'Collection',
 975             'required': True,
 976             'type': 'Directory',
 977         }
 978
 979     @stubs
 980     def test_create(self, stubs):
 981         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 982
 983         capture_stdout = cStringIO.StringIO()
 984
 985         exited = arvados_cwl.main(
 986             ["--create-workflow", "--debug",
 987              "--api=jobs",
 988              "--project-uuid", project_uuid,
 989              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 990             capture_stdout, sys.stderr, api_client=stubs.api)
 991         self.assertEqual(exited, 0)
 992
 993         stubs.api.pipeline_instances().create.refute_called()
 994         stubs.api.jobs().create.refute_called()
 995
 996         expect_component = copy.deepcopy(stubs.expect_job_spec)
 997         self._adjust_script_params(expect_component)
 998         expect_template = {
 999             "components": {
1000                 "submit_wf.cwl": expect_component,
1001             },
1002             "name": "submit_wf.cwl",
1003             "owner_uuid": project_uuid,
1004         }
1005         stubs.api.pipeline_templates().create.assert_called_with(
1006             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1007
1008         self.assertEqual(capture_stdout.getvalue(),
1009                          stubs.expect_pipeline_template_uuid + '\n')
1010
1011
1012     @stubs
1013     def test_create_name(self, stubs):
1014         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1015
1016         capture_stdout = cStringIO.StringIO()
1017
1018         exited = arvados_cwl.main(
1019             ["--create-workflow", "--debug",
1020              "--project-uuid", project_uuid,
1021              "--api=jobs",
1022              "--name", "testing 123",
1023              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1024             capture_stdout, sys.stderr, api_client=stubs.api)
1025         self.assertEqual(exited, 0)
1026
1027         stubs.api.pipeline_instances().create.refute_called()
1028         stubs.api.jobs().create.refute_called()
1029
1030         expect_component = copy.deepcopy(stubs.expect_job_spec)
1031         self._adjust_script_params(expect_component)
1032         expect_template = {
1033             "components": {
1034                 "testing 123": expect_component,
1035             },
1036             "name": "testing 123",
1037             "owner_uuid": project_uuid,
1038         }
1039         stubs.api.pipeline_templates().create.assert_called_with(
1040             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1041
1042         self.assertEqual(capture_stdout.getvalue(),
1043                          stubs.expect_pipeline_template_uuid + '\n')
1044
1045
1046     @stubs
1047     def test_update_name(self, stubs):
1048         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1049
1050         capture_stdout = cStringIO.StringIO()
1051
1052         exited = arvados_cwl.main(
1053             ["--update-workflow", self.existing_template_uuid,
1054              "--debug",
1055              "--project-uuid", project_uuid,
1056              "--api=jobs",
1057              "--name", "testing 123",
1058              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1059             capture_stdout, sys.stderr, api_client=stubs.api)
1060         self.assertEqual(exited, 0)
1061
1062         stubs.api.pipeline_instances().create.refute_called()
1063         stubs.api.jobs().create.refute_called()
1064
1065         expect_component = copy.deepcopy(stubs.expect_job_spec)
1066         self._adjust_script_params(expect_component)
1067         expect_template = {
1068             "components": {
1069                 "testing 123": expect_component,
1070             },
1071             "name": "testing 123",
1072             "owner_uuid": project_uuid,
1073         }
1074         stubs.api.pipeline_templates().create.refute_called()
1075         stubs.api.pipeline_templates().update.assert_called_with(
1076             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1077
1078         self.assertEqual(capture_stdout.getvalue(),
1079                          self.existing_template_uuid + '\n')
1080
1081
1082 class TestCreateWorkflow(unittest.TestCase):
1083     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1084     expect_workflow = StripYAMLComments(
1085         open("tests/wf/expect_packed.cwl").read())
1086
1087     @stubs
1088     def test_create(self, stubs):
1089         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1090
1091         capture_stdout = cStringIO.StringIO()
1092
1093         exited = arvados_cwl.main(
1094             ["--create-workflow", "--debug",
1095              "--api=containers",
1096              "--project-uuid", project_uuid,
1097              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1098             capture_stdout, sys.stderr, api_client=stubs.api)
1099         self.assertEqual(exited, 0)
1100
1101         stubs.api.pipeline_templates().create.refute_called()
1102         stubs.api.container_requests().create.refute_called()
1103
1104         body = {
1105             "workflow": {
1106                 "owner_uuid": project_uuid,
1107                 "name": "submit_wf.cwl",
1108                 "description": "",
1109                 "definition": self.expect_workflow,
1110             }
1111         }
1112         stubs.api.workflows().create.assert_called_with(
1113             body=JsonDiffMatcher(body))
1114
1115         self.assertEqual(capture_stdout.getvalue(),
1116                          stubs.expect_workflow_uuid + '\n')
1117
1118
1119     @stubs
1120     def test_create_name(self, stubs):
1121         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1122
1123         capture_stdout = cStringIO.StringIO()
1124
1125         exited = arvados_cwl.main(
1126             ["--create-workflow", "--debug",
1127              "--api=containers",
1128              "--project-uuid", project_uuid,
1129              "--name", "testing 123",
1130              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1131             capture_stdout, sys.stderr, api_client=stubs.api)
1132         self.assertEqual(exited, 0)
1133
1134         stubs.api.pipeline_templates().create.refute_called()
1135         stubs.api.container_requests().create.refute_called()
1136
1137         body = {
1138             "workflow": {
1139                 "owner_uuid": project_uuid,
1140                 "name": "testing 123",
1141                 "description": "",
1142                 "definition": self.expect_workflow,
1143             }
1144         }
1145         stubs.api.workflows().create.assert_called_with(
1146             body=JsonDiffMatcher(body))
1147
1148         self.assertEqual(capture_stdout.getvalue(),
1149                          stubs.expect_workflow_uuid + '\n')
1150
1151     @stubs
1152     def test_incompatible_api(self, stubs):
1153         capture_stderr = cStringIO.StringIO()
1154         logging.getLogger('arvados.cwl-runner').addHandler(
1155             logging.StreamHandler(capture_stderr))
1156
1157         exited = arvados_cwl.main(
1158             ["--update-workflow", self.existing_workflow_uuid,
1159              "--api=jobs",
1160              "--debug",
1161              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1162             sys.stderr, sys.stderr, api_client=stubs.api)
1163         self.assertEqual(exited, 1)
1164         self.assertRegexpMatches(
1165             capture_stderr.getvalue(),
1166             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1167
1168     @stubs
1169     def test_update(self, stubs):
1170         capture_stdout = cStringIO.StringIO()
1171
1172         exited = arvados_cwl.main(
1173             ["--update-workflow", self.existing_workflow_uuid,
1174              "--debug",
1175              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1176             capture_stdout, sys.stderr, api_client=stubs.api)
1177         self.assertEqual(exited, 0)
1178
1179         body = {
1180             "workflow": {
1181                 "name": "submit_wf.cwl",
1182                 "description": "",
1183                 "definition": self.expect_workflow,
1184             }
1185         }
1186         stubs.api.workflows().update.assert_called_with(
1187             uuid=self.existing_workflow_uuid,
1188             body=JsonDiffMatcher(body))
1189         self.assertEqual(capture_stdout.getvalue(),
1190                          self.existing_workflow_uuid + '\n')
1191
1192
1193     @stubs
1194     def test_update_name(self, stubs):
1195         capture_stdout = cStringIO.StringIO()
1196
1197         exited = arvados_cwl.main(
1198             ["--update-workflow", self.existing_workflow_uuid,
1199              "--debug", "--name", "testing 123",
1200              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1201             capture_stdout, sys.stderr, api_client=stubs.api)
1202         self.assertEqual(exited, 0)
1203
1204         body = {
1205             "workflow": {
1206                 "name": "testing 123",
1207                 "description": "",
1208                 "definition": self.expect_workflow,
1209             }
1210         }
1211         stubs.api.workflows().update.assert_called_with(
1212             uuid=self.existing_workflow_uuid,
1213             body=JsonDiffMatcher(body))
1214         self.assertEqual(capture_stdout.getvalue(),
1215                          self.existing_workflow_uuid + '\n')
1216
1217
1218     @stubs
1219     def test_create_collection_per_tool(self, stubs):
1220         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1221
1222         capture_stdout = cStringIO.StringIO()
1223
1224         exited = arvados_cwl.main(
1225             ["--create-workflow", "--debug",
1226              "--api=containers",
1227              "--project-uuid", project_uuid,
1228              "tests/collection_per_tool/collection_per_tool.cwl"],
1229             capture_stdout, sys.stderr, api_client=stubs.api)
1230         self.assertEqual(exited, 0)
1231
1232         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1233         expect_workflow = StripYAMLComments(open(toolfile).read())
1234
1235         body = {
1236             "workflow": {
1237                 "owner_uuid": project_uuid,
1238                 "name": "collection_per_tool.cwl",
1239                 "description": "",
1240                 "definition": expect_workflow,
1241             }
1242         }
1243         stubs.api.workflows().create.assert_called_with(
1244             body=JsonDiffMatcher(body))
1245
1246         self.assertEqual(capture_stdout.getvalue(),
1247                          stubs.expect_workflow_uuid + '\n')
1248
1249 class TestTemplateInputs(unittest.TestCase):
1250     expect_template = {
1251         "components": {
1252             "inputs_test.cwl": {
1253                 'runtime_constraints': {
1254                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1255                     'min_ram_mb_per_node': 1024
1256                 },
1257                 'script_parameters': {
1258                     'cwl:tool':
1259                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1260                     'optionalFloatInput': None,
1261                     'fileInput': {
1262                         'type': 'File',
1263                         'dataclass': 'File',
1264                         'required': True,
1265                         'title': "It's a file; we expect to find some characters in it.",
1266                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1267                     },
1268                     'floatInput': {
1269                         'type': 'float',
1270                         'dataclass': 'number',
1271                         'required': True,
1272                         'title': 'Floats like a duck',
1273                         'default': 0.1,
1274                         'value': 0.1,
1275                     },
1276                     'optionalFloatInput': {
1277                         'type': ['null', 'float'],
1278                         'dataclass': 'number',
1279                         'required': False,
1280                     },
1281                     'boolInput': {
1282                         'type': 'boolean',
1283                         'dataclass': 'boolean',
1284                         'required': True,
1285                         'title': 'True or false?',
1286                     },
1287                 },
1288                 'repository': 'arvados',
1289                 'script_version': 'master',
1290                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1291                 'script': 'cwl-runner',
1292             },
1293         },
1294         "name": "inputs_test.cwl",
1295     }
1296
1297     @stubs
1298     def test_inputs_empty(self, stubs):
1299         exited = arvados_cwl.main(
1300             ["--create-template",
1301              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1302             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1303         self.assertEqual(exited, 0)
1304
1305         stubs.api.pipeline_templates().create.assert_called_with(
1306             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1307
1308     @stubs
1309     def test_inputs(self, stubs):
1310         exited = arvados_cwl.main(
1311             ["--create-template",
1312              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1313             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1314         self.assertEqual(exited, 0)
1315
1316         expect_template = copy.deepcopy(self.expect_template)
1317         params = expect_template[
1318             "components"]["inputs_test.cwl"]["script_parameters"]
1319         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1320         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1321         params["floatInput"]["value"] = 1.234
1322         params["boolInput"]["value"] = True
1323
1324         stubs.api.pipeline_templates().create.assert_called_with(
1325             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)