sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {}
  83         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  84         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  85
  86         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  87         stubs.api.jobs().create().execute.return_value = {
  88             "uuid": stubs.expect_job_uuid,
  89             "state": "Queued",
  90         }
  91
  92         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  93         stubs.api.container_requests().create().execute.return_value = {
  94             "uuid": stubs.expect_container_request_uuid,
  95             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  96             "state": "Queued"
  97         }
  98
  99         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 100         stubs.api.pipeline_templates().create().execute.return_value = {
 101             "uuid": stubs.expect_pipeline_template_uuid,
 102         }
 103         stubs.expect_job_spec = {
 104             'runtime_constraints': {
 105                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 106                 'min_ram_mb_per_node': 1024
 107             },
 108             'script_parameters': {
 109                 'x': {
 110                     'basename': 'blorp.txt',
 111                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 112                     'class': 'File'
 113                 },
 114                 'y': {
 115                     'basename': '99999999999999999999999999999998+99',
 116                     'location': 'keep:99999999999999999999999999999998+99',
 117                     'class': 'Directory'
 118                 },
 119                 'z': {
 120                     'basename': 'anonymous',
 121                     "listing": [{
 122                         "basename": "renamed.txt",
 123                         "class": "File",
 124                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 125                     }],
 126                     'class': 'Directory'
 127                 },
 128                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 129             },
 130             'repository': 'arvados',
 131             'script_version': 'master',
 132             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 133             'script': 'cwl-runner'
 134         }
 135         stubs.pipeline_component = stubs.expect_job_spec.copy()
 136         stubs.expect_pipeline_instance = {
 137             'name': 'submit_wf.cwl',
 138             'state': 'RunningOnServer',
 139             'owner_uuid': None,
 140             "components": {
 141                 "cwl-runner": {
 142                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 143                     'script_parameters': {
 144                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 145                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
 146                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 147                               'listing': [
 148                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 149                               ]}},
 150                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 151                         'arv:enable_reuse': True,
 152                         'arv:on_error': 'continue'
 153                     },
 154                     'repository': 'arvados',
 155                     'script_version': 'master',
 156                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 157                     'script': 'cwl-runner',
 158                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 159                 }
 160             }
 161         }
 162         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 163         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 164         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 165         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 166         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 167             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 168             "state": "Queued"
 169         }
 170         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 171         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 172
 173         with open("tests/wf/submit_wf_packed.cwl") as f:
 174             expect_packed_workflow = yaml.round_trip_load(f)
 175
 176         stubs.expect_container_spec = {
 177             'priority': 1,
 178             'mounts': {
 179                 '/var/spool/cwl': {
 180                     'writable': True,
 181                     'kind': 'collection'
 182                 },
 183                 '/var/lib/cwl/workflow.json': {
 184                     'content': expect_packed_workflow,
 185                     'kind': 'json'
 186                 },
 187                 'stdout': {
 188                     'path': '/var/spool/cwl/cwl.output.json',
 189                     'kind': 'file'
 190                 },
 191                 '/var/lib/cwl/cwl.input.json': {
 192                     'kind': 'json',
 193                     'content': {
 194                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 195                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
 196                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 197                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 198                         ]}
 199                     },
 200                     'kind': 'json'
 201                 }
 202             },
 203             'state': 'Committed',
 204             'owner_uuid': None,
 205             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 206                         '--enable-reuse', '--on-error=continue',
 207                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 208             'name': 'submit_wf.cwl',
 209             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 210             'output_path': '/var/spool/cwl',
 211             'cwd': '/var/spool/cwl',
 212             'runtime_constraints': {
 213                 'API': True,
 214                 'vcpus': 1,
 215                 'ram': 1024*1024*1024
 216             },
 217             "properties": {}
 218         }
 219
 220         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 221         stubs.api.workflows().create().execute.return_value = {
 222             "uuid": stubs.expect_workflow_uuid,
 223         }
 224         def update_mock(**kwargs):
 225             stubs.updated_uuid = kwargs.get('uuid')
 226             return mock.DEFAULT
 227         stubs.api.workflows().update.side_effect = update_mock
 228         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 229             "uuid": stubs.updated_uuid,
 230         }
 231
 232         return func(self, stubs, *args, **kwargs)
 233     return wrapped
 234
 235
 236 class TestSubmit(unittest.TestCase):
 237     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 238     @mock.patch("time.sleep")
 239     @stubs
 240     def test_submit(self, stubs, tm, arvdock):
 241         capture_stdout = cStringIO.StringIO()
 242         exited = arvados_cwl.main(
 243             ["--submit", "--no-wait", "--api=jobs", "--debug",
 244              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 245             capture_stdout, sys.stderr, api_client=stubs.api)
 246         self.assertEqual(exited, 0)
 247
 248         stubs.api.collections().create.assert_has_calls([
 249             mock.call(body=JsonDiffMatcher({
 250                 'manifest_text':
 251                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 252                 'replication_desired': None,
 253                 'name': 'submit_tool.cwl dependencies',
 254             }), ensure_unique_name=True),
 255             mock.call(body=JsonDiffMatcher({
 256                 'manifest_text':
 257                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 258                 'replication_desired': None,
 259                 'name': 'submit_wf.cwl input',
 260             }), ensure_unique_name=True),
 261             mock.call(body=JsonDiffMatcher({
 262                 'manifest_text':
 263                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 264                 'replication_desired': None,
 265                 'name': 'submit_wf.cwl',
 266             }), ensure_unique_name=True)        ])
 267
 268         arvdock.assert_has_calls([
 269             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 270             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 271         ])
 272
 273         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 274         stubs.api.pipeline_instances().create.assert_called_with(
 275             body=JsonDiffMatcher(expect_pipeline))
 276         self.assertEqual(capture_stdout.getvalue(),
 277                          stubs.expect_pipeline_uuid + '\n')
 278
 279
 280     @mock.patch("time.sleep")
 281     @stubs
 282     def test_submit_no_reuse(self, stubs, tm):
 283         capture_stdout = cStringIO.StringIO()
 284         exited = arvados_cwl.main(
 285             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 286              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 287             capture_stdout, sys.stderr, api_client=stubs.api)
 288         self.assertEqual(exited, 0)
 289
 290         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 291         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 292
 293         stubs.api.pipeline_instances().create.assert_called_with(
 294             body=JsonDiffMatcher(expect_pipeline))
 295         self.assertEqual(capture_stdout.getvalue(),
 296                          stubs.expect_pipeline_uuid + '\n')
 297
 298     @mock.patch("time.sleep")
 299     @stubs
 300     def test_submit_on_error(self, stubs, tm):
 301         capture_stdout = cStringIO.StringIO()
 302         exited = arvados_cwl.main(
 303             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 304              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 305             capture_stdout, sys.stderr, api_client=stubs.api)
 306         self.assertEqual(exited, 0)
 307
 308         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 309         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 310
 311         stubs.api.pipeline_instances().create.assert_called_with(
 312             body=JsonDiffMatcher(expect_pipeline))
 313         self.assertEqual(capture_stdout.getvalue(),
 314                          stubs.expect_pipeline_uuid + '\n')
 315
 316
 317     @mock.patch("time.sleep")
 318     @stubs
 319     def test_submit_runner_ram(self, stubs, tm):
 320         capture_stdout = cStringIO.StringIO()
 321         exited = arvados_cwl.main(
 322             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 323              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 324             capture_stdout, sys.stderr, api_client=stubs.api)
 325         self.assertEqual(exited, 0)
 326
 327         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 328         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 329
 330         stubs.api.pipeline_instances().create.assert_called_with(
 331             body=JsonDiffMatcher(expect_pipeline))
 332         self.assertEqual(capture_stdout.getvalue(),
 333                          stubs.expect_pipeline_uuid + '\n')
 334
 335
 336     @mock.patch("time.sleep")
 337     @stubs
 338     def test_submit_invalid_runner_ram(self, stubs, tm):
 339         capture_stdout = cStringIO.StringIO()
 340         exited = arvados_cwl.main(
 341             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 342              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 343             capture_stdout, sys.stderr, api_client=stubs.api)
 344         self.assertEqual(exited, 1)
 345
 346     @mock.patch("time.sleep")
 347     @stubs
 348     def test_submit_output_name(self, stubs, tm):
 349         output_name = "test_output_name"
 350
 351         capture_stdout = cStringIO.StringIO()
 352         exited = arvados_cwl.main(
 353             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 354              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 355             capture_stdout, sys.stderr, api_client=stubs.api)
 356         self.assertEqual(exited, 0)
 357
 358         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 359         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 360
 361         stubs.api.pipeline_instances().create.assert_called_with(
 362             body=JsonDiffMatcher(expect_pipeline))
 363         self.assertEqual(capture_stdout.getvalue(),
 364                          stubs.expect_pipeline_uuid + '\n')
 365
 366
 367     @mock.patch("time.sleep")
 368     @stubs
 369     def test_submit_pipeline_name(self, stubs, tm):
 370         capture_stdout = cStringIO.StringIO()
 371         exited = arvados_cwl.main(
 372             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 373              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 374             capture_stdout, sys.stderr, api_client=stubs.api)
 375         self.assertEqual(exited, 0)
 376
 377         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 378         expect_pipeline["name"] = "hello job 123"
 379
 380         stubs.api.pipeline_instances().create.assert_called_with(
 381             body=JsonDiffMatcher(expect_pipeline))
 382         self.assertEqual(capture_stdout.getvalue(),
 383                          stubs.expect_pipeline_uuid + '\n')
 384
 385     @mock.patch("time.sleep")
 386     @stubs
 387     def test_submit_output_tags(self, stubs, tm):
 388         output_tags = "tag0,tag1,tag2"
 389
 390         capture_stdout = cStringIO.StringIO()
 391         exited = arvados_cwl.main(
 392             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 393              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 394             capture_stdout, sys.stderr, api_client=stubs.api)
 395         self.assertEqual(exited, 0)
 396
 397         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 398         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 399
 400         stubs.api.pipeline_instances().create.assert_called_with(
 401             body=JsonDiffMatcher(expect_pipeline))
 402         self.assertEqual(capture_stdout.getvalue(),
 403                          stubs.expect_pipeline_uuid + '\n')
 404
 405     @mock.patch("time.sleep")
 406     @stubs
 407     def test_submit_with_project_uuid(self, stubs, tm):
 408         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 409
 410         exited = arvados_cwl.main(
 411             ["--submit", "--no-wait",
 412              "--project-uuid", project_uuid,
 413              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 414             sys.stdout, sys.stderr, api_client=stubs.api)
 415         self.assertEqual(exited, 0)
 416
 417         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 418         expect_pipeline["owner_uuid"] = project_uuid
 419         stubs.api.pipeline_instances().create.assert_called_with(
 420             body=JsonDiffMatcher(expect_pipeline))
 421
 422     @stubs
 423     def test_submit_container(self, stubs):
 424         capture_stdout = cStringIO.StringIO()
 425         try:
 426             exited = arvados_cwl.main(
 427                 ["--submit", "--no-wait", "--api=containers", "--debug",
 428                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 429                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 430             self.assertEqual(exited, 0)
 431         except:
 432             logging.exception("")
 433
 434         stubs.api.collections().create.assert_has_calls([
 435             mock.call(body=JsonDiffMatcher({
 436                 'manifest_text':
 437                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 438                 'replication_desired': None,
 439                 'name': 'submit_tool.cwl dependencies',
 440             }), ensure_unique_name=True),
 441             mock.call(body=JsonDiffMatcher({
 442                 'manifest_text':
 443                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 444                 'replication_desired': None,
 445                 'name': 'submit_wf.cwl input',
 446             }), ensure_unique_name=True)])
 447
 448         expect_container = copy.deepcopy(stubs.expect_container_spec)
 449         stubs.api.container_requests().create.assert_called_with(
 450             body=JsonDiffMatcher(expect_container))
 451         self.assertEqual(capture_stdout.getvalue(),
 452                          stubs.expect_container_request_uuid + '\n')
 453
 454     @stubs
 455     def test_submit_container_no_reuse(self, stubs):
 456         capture_stdout = cStringIO.StringIO()
 457         try:
 458             exited = arvados_cwl.main(
 459                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 460                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 461                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 462             self.assertEqual(exited, 0)
 463         except:
 464             logging.exception("")
 465
 466         expect_container = copy.deepcopy(stubs.expect_container_spec)
 467         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 468                                                   '--disable-reuse', '--on-error=continue',
 469                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 470
 471         stubs.api.container_requests().create.assert_called_with(
 472             body=JsonDiffMatcher(expect_container))
 473         self.assertEqual(capture_stdout.getvalue(),
 474                          stubs.expect_container_request_uuid + '\n')
 475
 476
 477     @stubs
 478     def test_submit_container_on_error(self, stubs):
 479         capture_stdout = cStringIO.StringIO()
 480         try:
 481             exited = arvados_cwl.main(
 482                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 483                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 484                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 485             self.assertEqual(exited, 0)
 486         except:
 487             logging.exception("")
 488
 489         expect_container = copy.deepcopy(stubs.expect_container_spec)
 490         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 491                                                   '--enable-reuse', '--on-error=stop',
 492                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 493
 494         stubs.api.container_requests().create.assert_called_with(
 495             body=JsonDiffMatcher(expect_container))
 496         self.assertEqual(capture_stdout.getvalue(),
 497                          stubs.expect_container_request_uuid + '\n')
 498
 499     @stubs
 500     def test_submit_container_output_name(self, stubs):
 501         output_name = "test_output_name"
 502
 503         capture_stdout = cStringIO.StringIO()
 504         try:
 505             exited = arvados_cwl.main(
 506                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 507                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 508                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 509             self.assertEqual(exited, 0)
 510         except:
 511             logging.exception("")
 512
 513         expect_container = copy.deepcopy(stubs.expect_container_spec)
 514         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 515                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 516                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 517         expect_container["output_name"] = output_name
 518
 519         stubs.api.container_requests().create.assert_called_with(
 520             body=JsonDiffMatcher(expect_container))
 521         self.assertEqual(capture_stdout.getvalue(),
 522                          stubs.expect_container_request_uuid + '\n')
 523
 524
 525     @stubs
 526     def test_submit_container_output_ttl(self, stubs):
 527         capture_stdout = cStringIO.StringIO()
 528         try:
 529             exited = arvados_cwl.main(
 530                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 531                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533             self.assertEqual(exited, 0)
 534         except:
 535             logging.exception("")
 536
 537         expect_container = copy.deepcopy(stubs.expect_container_spec)
 538         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 539                                        '--enable-reuse', '--on-error=continue',
 540                                        "--intermediate-output-ttl=3600",
 541                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 542
 543         stubs.api.container_requests().create.assert_called_with(
 544             body=JsonDiffMatcher(expect_container))
 545         self.assertEqual(capture_stdout.getvalue(),
 546                          stubs.expect_container_request_uuid + '\n')
 547
 548     @stubs
 549     def test_submit_container_trash_intermediate(self, stubs):
 550         capture_stdout = cStringIO.StringIO()
 551         try:
 552             exited = arvados_cwl.main(
 553                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 554                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 555                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 556             self.assertEqual(exited, 0)
 557         except:
 558             logging.exception("")
 559
 560         expect_container = copy.deepcopy(stubs.expect_container_spec)
 561         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 562                                        '--enable-reuse', '--on-error=continue',
 563                                        "--trash-intermediate",
 564                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 565
 566         stubs.api.container_requests().create.assert_called_with(
 567             body=JsonDiffMatcher(expect_container))
 568         self.assertEqual(capture_stdout.getvalue(),
 569                          stubs.expect_container_request_uuid + '\n')
 570
 571     @stubs
 572     def test_submit_container_output_tags(self, stubs):
 573         output_tags = "tag0,tag1,tag2"
 574
 575         capture_stdout = cStringIO.StringIO()
 576         try:
 577             exited = arvados_cwl.main(
 578                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 579                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 580                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 581             self.assertEqual(exited, 0)
 582         except:
 583             logging.exception("")
 584
 585         expect_container = copy.deepcopy(stubs.expect_container_spec)
 586         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 587                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 588                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 589
 590         stubs.api.container_requests().create.assert_called_with(
 591             body=JsonDiffMatcher(expect_container))
 592         self.assertEqual(capture_stdout.getvalue(),
 593                          stubs.expect_container_request_uuid + '\n')
 594
 595     @stubs
 596     def test_submit_container_runner_ram(self, stubs):
 597         capture_stdout = cStringIO.StringIO()
 598         try:
 599             exited = arvados_cwl.main(
 600                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 601                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 602                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 603             self.assertEqual(exited, 0)
 604         except:
 605             logging.exception("")
 606
 607         expect_container = copy.deepcopy(stubs.expect_container_spec)
 608         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 609
 610         stubs.api.container_requests().create.assert_called_with(
 611             body=JsonDiffMatcher(expect_container))
 612         self.assertEqual(capture_stdout.getvalue(),
 613                          stubs.expect_container_request_uuid + '\n')
 614
 615     @mock.patch("arvados.collection.CollectionReader")
 616     @mock.patch("time.sleep")
 617     @stubs
 618     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 619         capture_stdout = cStringIO.StringIO()
 620         exited = arvados_cwl.main(
 621             ["--submit", "--no-wait", "--api=containers", "--debug",
 622              "tests/wf/submit_keepref_wf.cwl"],
 623             capture_stdout, sys.stderr, api_client=stubs.api)
 624         self.assertEqual(exited, 0)
 625
 626
 627     @mock.patch("arvados.collection.CollectionReader")
 628     @mock.patch("time.sleep")
 629     @stubs
 630     def test_submit_keepref(self, stubs, tm, reader):
 631         capture_stdout = cStringIO.StringIO()
 632
 633         with open("tests/wf/expect_arvworkflow.cwl") as f:
 634             reader().open().__enter__().read.return_value = f.read()
 635
 636         exited = arvados_cwl.main(
 637             ["--submit", "--no-wait", "--api=containers", "--debug",
 638              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 639             capture_stdout, sys.stderr, api_client=stubs.api)
 640         self.assertEqual(exited, 0)
 641
 642         expect_container = {
 643             'priority': 1,
 644             'mounts': {
 645                 '/var/spool/cwl': {
 646                     'writable': True,
 647                     'kind': 'collection'
 648                 },
 649                 'stdout': {
 650                     'path': '/var/spool/cwl/cwl.output.json',
 651                     'kind': 'file'
 652                 },
 653                 '/var/lib/cwl/workflow': {
 654                     'portable_data_hash': '99999999999999999999999999999994+99',
 655                     'kind': 'collection'
 656                 },
 657                 '/var/lib/cwl/cwl.input.json': {
 658                     'content': {
 659                         'x': 'XxX'
 660                     },
 661                     'kind': 'json'
 662                 }
 663             }, 'state': 'Committed',
 664             'owner_uuid': None,
 665             'output_path': '/var/spool/cwl',
 666             'name': 'expect_arvworkflow.cwl#main',
 667             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 668             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 669                         '--enable-reuse', '--on-error=continue',
 670                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 671             'cwd': '/var/spool/cwl',
 672             'runtime_constraints': {
 673                 'API': True,
 674                 'vcpus': 1,
 675                 'ram': 1073741824
 676             },
 677             "properties": {}
 678         }
 679
 680         stubs.api.container_requests().create.assert_called_with(
 681             body=JsonDiffMatcher(expect_container))
 682         self.assertEqual(capture_stdout.getvalue(),
 683                          stubs.expect_container_request_uuid + '\n')
 684
 685
 686     @mock.patch("arvados.collection.CollectionReader")
 687     @mock.patch("time.sleep")
 688     @stubs
 689     def test_submit_jobs_keepref(self, stubs, tm, reader):
 690         capture_stdout = cStringIO.StringIO()
 691
 692         with open("tests/wf/expect_arvworkflow.cwl") as f:
 693             reader().open().__enter__().read.return_value = f.read()
 694
 695         exited = arvados_cwl.main(
 696             ["--submit", "--no-wait", "--api=jobs", "--debug",
 697              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 698             capture_stdout, sys.stderr, api_client=stubs.api)
 699         self.assertEqual(exited, 0)
 700
 701         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 702         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 703         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 704         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 705         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 706         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 707         stubs.api.pipeline_instances().create.assert_called_with(
 708             body=JsonDiffMatcher(expect_pipeline))
 709
 710     @mock.patch("time.sleep")
 711     @stubs
 712     def test_submit_arvworkflow(self, stubs, tm):
 713         capture_stdout = cStringIO.StringIO()
 714
 715         with open("tests/wf/expect_arvworkflow.cwl") as f:
 716             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 717
 718         exited = arvados_cwl.main(
 719             ["--submit", "--no-wait", "--api=containers", "--debug",
 720              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 721             capture_stdout, sys.stderr, api_client=stubs.api)
 722         self.assertEqual(exited, 0)
 723
 724         expect_container = {
 725             'priority': 1,
 726             'mounts': {
 727                 '/var/spool/cwl': {
 728                     'writable': True,
 729                     'kind': 'collection'
 730                 },
 731                 'stdout': {
 732                     'path': '/var/spool/cwl/cwl.output.json',
 733                     'kind': 'file'
 734                 },
 735                 '/var/lib/cwl/workflow.json': {
 736                     'kind': 'json',
 737                     'content': {
 738                         'cwlVersion': 'v1.0',
 739                         '$graph': [
 740                             {
 741                                 'id': '#main',
 742                                 'inputs': [
 743                                     {'type': 'string', 'id': '#main/x'}
 744                                 ],
 745                                 'steps': [
 746                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 747                                      'run': '#submit_tool.cwl',
 748                                      'id': '#main/step1',
 749                                      'out': []}
 750                                 ],
 751                                 'class': 'Workflow',
 752                                 'outputs': []
 753                             },
 754                             {
 755                                 'inputs': [
 756                                     {
 757                                         'inputBinding': {'position': 1},
 758                                         'type': 'string',
 759                                         'id': '#submit_tool.cwl/x'}
 760                                 ],
 761                                 'requirements': [
 762                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 763                                 ],
 764                                 'id': '#submit_tool.cwl',
 765                                 'outputs': [],
 766                                 'baseCommand': 'cat',
 767                                 'class': 'CommandLineTool'
 768                             }
 769                         ]
 770                     }
 771                 },
 772                 '/var/lib/cwl/cwl.input.json': {
 773                     'content': {
 774                         'x': 'XxX'
 775                     },
 776                     'kind': 'json'
 777                 }
 778             }, 'state': 'Committed',
 779             'owner_uuid': None,
 780             'output_path': '/var/spool/cwl',
 781             'name': 'a test workflow',
 782             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 783             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 784                         '--enable-reuse', '--on-error=continue',
 785                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 786             'cwd': '/var/spool/cwl',
 787             'runtime_constraints': {
 788                 'API': True,
 789                 'vcpus': 1,
 790                 'ram': 1073741824
 791             },
 792             "properties": {
 793                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 794             }
 795         }
 796
 797         stubs.api.container_requests().create.assert_called_with(
 798             body=JsonDiffMatcher(expect_container))
 799         self.assertEqual(capture_stdout.getvalue(),
 800                          stubs.expect_container_request_uuid + '\n')
 801
 802
 803     @stubs
 804     def test_submit_container_name(self, stubs):
 805         capture_stdout = cStringIO.StringIO()
 806         try:
 807             exited = arvados_cwl.main(
 808                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 809                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 810                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 811             self.assertEqual(exited, 0)
 812         except:
 813             logging.exception("")
 814
 815         expect_container = copy.deepcopy(stubs.expect_container_spec)
 816         expect_container["name"] = "hello container 123"
 817
 818         stubs.api.container_requests().create.assert_called_with(
 819             body=JsonDiffMatcher(expect_container))
 820         self.assertEqual(capture_stdout.getvalue(),
 821                          stubs.expect_container_request_uuid + '\n')
 822
 823
 824     @stubs
 825     def test_submit_job_runner_image(self, stubs):
 826         capture_stdout = cStringIO.StringIO()
 827         try:
 828             exited = arvados_cwl.main(
 829                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 830                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 831                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 832             self.assertEqual(exited, 0)
 833         except:
 834             logging.exception("")
 835
 836         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 837
 838         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 839         stubs.api.pipeline_instances().create.assert_called_with(
 840             body=JsonDiffMatcher(expect_pipeline))
 841         self.assertEqual(capture_stdout.getvalue(),
 842                          stubs.expect_pipeline_uuid + '\n')
 843
 844     @stubs
 845     def test_submit_container_runner_image(self, stubs):
 846         capture_stdout = cStringIO.StringIO()
 847         try:
 848             exited = arvados_cwl.main(
 849                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 850                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 851                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 852             self.assertEqual(exited, 0)
 853         except:
 854             logging.exception("")
 855
 856         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 857
 858         expect_container = copy.deepcopy(stubs.expect_container_spec)
 859         stubs.api.container_requests().create.assert_called_with(
 860             body=JsonDiffMatcher(expect_container))
 861         self.assertEqual(capture_stdout.getvalue(),
 862                          stubs.expect_container_request_uuid + '\n')
 863
 864
 865     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 866     @mock.patch("cwltool.docker.get_image")
 867     @mock.patch("arvados.api")
 868     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 869         arvrunner = mock.MagicMock()
 870         arvrunner.project_uuid = ""
 871         api.return_value = mock.MagicMock()
 872         arvrunner.api = api.return_value
 873         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 874                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 875                                                                         "link_class": "docker_image_repo+tag",
 876                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 877                                                                         "owner_uuid": "",
 878                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 879                                                             {"items": [{"created_at": "",
 880                                                                         "head_uuid": "",
 881                                                                         "link_class": "docker_image_hash",
 882                                                                         "name": "123456",
 883                                                                         "owner_uuid": "",
 884                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 885         )
 886         find_one_image_hash.return_value = "123456"
 887
 888         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 889                                                                               "owner_uuid": "",
 890                                                                               "manifest_text": "",
 891                                                                               "properties": ""
 892                                                                           }], "items_available": 1, "offset": 0},)
 893         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 894         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 895                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 896
 897 class TestCreateTemplate(unittest.TestCase):
 898     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 899
 900     def _adjust_script_params(self, expect_component):
 901         expect_component['script_parameters']['x'] = {
 902             'dataclass': 'File',
 903             'required': True,
 904             'type': 'File',
 905             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 906         }
 907         expect_component['script_parameters']['y'] = {
 908             'dataclass': 'Collection',
 909             'required': True,
 910             'type': 'Directory',
 911             'value': '99999999999999999999999999999998+99',
 912         }
 913         expect_component['script_parameters']['z'] = {
 914             'dataclass': 'Collection',
 915             'required': True,
 916             'type': 'Directory',
 917         }
 918
 919     @stubs
 920     def test_create(self, stubs):
 921         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 922
 923         capture_stdout = cStringIO.StringIO()
 924
 925         exited = arvados_cwl.main(
 926             ["--create-workflow", "--debug",
 927              "--api=jobs",
 928              "--project-uuid", project_uuid,
 929              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 930             capture_stdout, sys.stderr, api_client=stubs.api)
 931         self.assertEqual(exited, 0)
 932
 933         stubs.api.pipeline_instances().create.refute_called()
 934         stubs.api.jobs().create.refute_called()
 935
 936         expect_component = copy.deepcopy(stubs.expect_job_spec)
 937         self._adjust_script_params(expect_component)
 938         expect_template = {
 939             "components": {
 940                 "submit_wf.cwl": expect_component,
 941             },
 942             "name": "submit_wf.cwl",
 943             "owner_uuid": project_uuid,
 944         }
 945         stubs.api.pipeline_templates().create.assert_called_with(
 946             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 947
 948         self.assertEqual(capture_stdout.getvalue(),
 949                          stubs.expect_pipeline_template_uuid + '\n')
 950
 951
 952     @stubs
 953     def test_create_name(self, stubs):
 954         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 955
 956         capture_stdout = cStringIO.StringIO()
 957
 958         exited = arvados_cwl.main(
 959             ["--create-workflow", "--debug",
 960              "--project-uuid", project_uuid,
 961              "--api=jobs",
 962              "--name", "testing 123",
 963              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 964             capture_stdout, sys.stderr, api_client=stubs.api)
 965         self.assertEqual(exited, 0)
 966
 967         stubs.api.pipeline_instances().create.refute_called()
 968         stubs.api.jobs().create.refute_called()
 969
 970         expect_component = copy.deepcopy(stubs.expect_job_spec)
 971         self._adjust_script_params(expect_component)
 972         expect_template = {
 973             "components": {
 974                 "testing 123": expect_component,
 975             },
 976             "name": "testing 123",
 977             "owner_uuid": project_uuid,
 978         }
 979         stubs.api.pipeline_templates().create.assert_called_with(
 980             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 981
 982         self.assertEqual(capture_stdout.getvalue(),
 983                          stubs.expect_pipeline_template_uuid + '\n')
 984
 985
 986     @stubs
 987     def test_update_name(self, stubs):
 988         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 989
 990         capture_stdout = cStringIO.StringIO()
 991
 992         exited = arvados_cwl.main(
 993             ["--update-workflow", self.existing_template_uuid,
 994              "--debug",
 995              "--project-uuid", project_uuid,
 996              "--api=jobs",
 997              "--name", "testing 123",
 998              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 999             capture_stdout, sys.stderr, api_client=stubs.api)
1000         self.assertEqual(exited, 0)
1001
1002         stubs.api.pipeline_instances().create.refute_called()
1003         stubs.api.jobs().create.refute_called()
1004
1005         expect_component = copy.deepcopy(stubs.expect_job_spec)
1006         self._adjust_script_params(expect_component)
1007         expect_template = {
1008             "components": {
1009                 "testing 123": expect_component,
1010             },
1011             "name": "testing 123",
1012             "owner_uuid": project_uuid,
1013         }
1014         stubs.api.pipeline_templates().create.refute_called()
1015         stubs.api.pipeline_templates().update.assert_called_with(
1016             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1017
1018         self.assertEqual(capture_stdout.getvalue(),
1019                          self.existing_template_uuid + '\n')
1020
1021
1022 class TestCreateWorkflow(unittest.TestCase):
1023     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1024     expect_workflow = StripYAMLComments(
1025         open("tests/wf/expect_packed.cwl").read())
1026
1027     @stubs
1028     def test_create(self, stubs):
1029         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1030
1031         capture_stdout = cStringIO.StringIO()
1032
1033         exited = arvados_cwl.main(
1034             ["--create-workflow", "--debug",
1035              "--api=containers",
1036              "--project-uuid", project_uuid,
1037              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1038             capture_stdout, sys.stderr, api_client=stubs.api)
1039         self.assertEqual(exited, 0)
1040
1041         stubs.api.pipeline_templates().create.refute_called()
1042         stubs.api.container_requests().create.refute_called()
1043
1044         body = {
1045             "workflow": {
1046                 "owner_uuid": project_uuid,
1047                 "name": "submit_wf.cwl",
1048                 "description": "",
1049                 "definition": self.expect_workflow,
1050             }
1051         }
1052         stubs.api.workflows().create.assert_called_with(
1053             body=JsonDiffMatcher(body))
1054
1055         self.assertEqual(capture_stdout.getvalue(),
1056                          stubs.expect_workflow_uuid + '\n')
1057
1058
1059     @stubs
1060     def test_create_name(self, stubs):
1061         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1062
1063         capture_stdout = cStringIO.StringIO()
1064
1065         exited = arvados_cwl.main(
1066             ["--create-workflow", "--debug",
1067              "--api=containers",
1068              "--project-uuid", project_uuid,
1069              "--name", "testing 123",
1070              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1071             capture_stdout, sys.stderr, api_client=stubs.api)
1072         self.assertEqual(exited, 0)
1073
1074         stubs.api.pipeline_templates().create.refute_called()
1075         stubs.api.container_requests().create.refute_called()
1076
1077         body = {
1078             "workflow": {
1079                 "owner_uuid": project_uuid,
1080                 "name": "testing 123",
1081                 "description": "",
1082                 "definition": self.expect_workflow,
1083             }
1084         }
1085         stubs.api.workflows().create.assert_called_with(
1086             body=JsonDiffMatcher(body))
1087
1088         self.assertEqual(capture_stdout.getvalue(),
1089                          stubs.expect_workflow_uuid + '\n')
1090
1091     @stubs
1092     def test_incompatible_api(self, stubs):
1093         capture_stderr = cStringIO.StringIO()
1094         logging.getLogger('arvados.cwl-runner').addHandler(
1095             logging.StreamHandler(capture_stderr))
1096
1097         exited = arvados_cwl.main(
1098             ["--update-workflow", self.existing_workflow_uuid,
1099              "--api=jobs",
1100              "--debug",
1101              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1102             sys.stderr, sys.stderr, api_client=stubs.api)
1103         self.assertEqual(exited, 1)
1104         self.assertRegexpMatches(
1105             capture_stderr.getvalue(),
1106             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1107
1108     @stubs
1109     def test_update(self, stubs):
1110         capture_stdout = cStringIO.StringIO()
1111
1112         exited = arvados_cwl.main(
1113             ["--update-workflow", self.existing_workflow_uuid,
1114              "--debug",
1115              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1116             capture_stdout, sys.stderr, api_client=stubs.api)
1117         self.assertEqual(exited, 0)
1118
1119         body = {
1120             "workflow": {
1121                 "name": "submit_wf.cwl",
1122                 "description": "",
1123                 "definition": self.expect_workflow,
1124             }
1125         }
1126         stubs.api.workflows().update.assert_called_with(
1127             uuid=self.existing_workflow_uuid,
1128             body=JsonDiffMatcher(body))
1129         self.assertEqual(capture_stdout.getvalue(),
1130                          self.existing_workflow_uuid + '\n')
1131
1132
1133     @stubs
1134     def test_update_name(self, stubs):
1135         capture_stdout = cStringIO.StringIO()
1136
1137         exited = arvados_cwl.main(
1138             ["--update-workflow", self.existing_workflow_uuid,
1139              "--debug", "--name", "testing 123",
1140              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1141             capture_stdout, sys.stderr, api_client=stubs.api)
1142         self.assertEqual(exited, 0)
1143
1144         body = {
1145             "workflow": {
1146                 "name": "testing 123",
1147                 "description": "",
1148                 "definition": self.expect_workflow,
1149             }
1150         }
1151         stubs.api.workflows().update.assert_called_with(
1152             uuid=self.existing_workflow_uuid,
1153             body=JsonDiffMatcher(body))
1154         self.assertEqual(capture_stdout.getvalue(),
1155                          self.existing_workflow_uuid + '\n')
1156
1157
1158     @stubs
1159     def test_create_collection_per_tool(self, stubs):
1160         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1161
1162         capture_stdout = cStringIO.StringIO()
1163
1164         exited = arvados_cwl.main(
1165             ["--create-workflow", "--debug",
1166              "--api=containers",
1167              "--project-uuid", project_uuid,
1168              "tests/collection_per_tool/collection_per_tool.cwl"],
1169             capture_stdout, sys.stderr, api_client=stubs.api)
1170         self.assertEqual(exited, 0)
1171
1172         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1173         expect_workflow = StripYAMLComments(open(toolfile).read())
1174
1175         body = {
1176             "workflow": {
1177                 "owner_uuid": project_uuid,
1178                 "name": "collection_per_tool.cwl",
1179                 "description": "",
1180                 "definition": expect_workflow,
1181             }
1182         }
1183         stubs.api.workflows().create.assert_called_with(
1184             body=JsonDiffMatcher(body))
1185
1186         self.assertEqual(capture_stdout.getvalue(),
1187                          stubs.expect_workflow_uuid + '\n')
1188
1189 class TestTemplateInputs(unittest.TestCase):
1190     expect_template = {
1191         "components": {
1192             "inputs_test.cwl": {
1193                 'runtime_constraints': {
1194                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1195                     'min_ram_mb_per_node': 1024
1196                 },
1197                 'script_parameters': {
1198                     'cwl:tool':
1199                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1200                     'optionalFloatInput': None,
1201                     'fileInput': {
1202                         'type': 'File',
1203                         'dataclass': 'File',
1204                         'required': True,
1205                         'title': "It's a file; we expect to find some characters in it.",
1206                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1207                     },
1208                     'floatInput': {
1209                         'type': 'float',
1210                         'dataclass': 'number',
1211                         'required': True,
1212                         'title': 'Floats like a duck',
1213                         'default': 0.1,
1214                         'value': 0.1,
1215                     },
1216                     'optionalFloatInput': {
1217                         'type': ['null', 'float'],
1218                         'dataclass': 'number',
1219                         'required': False,
1220                     },
1221                     'boolInput': {
1222                         'type': 'boolean',
1223                         'dataclass': 'boolean',
1224                         'required': True,
1225                         'title': 'True or false?',
1226                     },
1227                 },
1228                 'repository': 'arvados',
1229                 'script_version': 'master',
1230                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1231                 'script': 'cwl-runner',
1232             },
1233         },
1234         "name": "inputs_test.cwl",
1235     }
1236
1237     @stubs
1238     def test_inputs_empty(self, stubs):
1239         exited = arvados_cwl.main(
1240             ["--create-template",
1241              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1242             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1243         self.assertEqual(exited, 0)
1244
1245         stubs.api.pipeline_templates().create.assert_called_with(
1246             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1247
1248     @stubs
1249     def test_inputs(self, stubs):
1250         exited = arvados_cwl.main(
1251             ["--create-template",
1252              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1253             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1254         self.assertEqual(exited, 0)
1255
1256         expect_template = copy.deepcopy(self.expect_template)
1257         params = expect_template[
1258             "components"]["inputs_test.cwl"]["script_parameters"]
1259         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1260         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1261         params["floatInput"]["value"] = 1.234
1262         params["boolInput"]["value"] = True
1263
1264         stubs.api.pipeline_templates().create.assert_called_with(
1265             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)