sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 import ruamel.yaml as yaml
  21
  22 _rootDesc = None
  23
  24 def stubs(func):
  25     @functools.wraps(func)
  26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  27     @mock.patch("arvados.collection.KeepClient")
  28     @mock.patch("arvados.keep.KeepClient")
  29     @mock.patch("arvados.events.subscribe")
  30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  31         class Stubs:
  32             pass
  33         stubs = Stubs()
  34         stubs.events = events
  35         stubs.keepdocker = keepdocker
  36
  37         def putstub(p, **kwargs):
  38             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  39         keep_client1().put.side_effect = putstub
  40         keep_client1.put.side_effect = putstub
  41         keep_client2().put.side_effect = putstub
  42         keep_client2.put.side_effect = putstub
  43
  44         stubs.keep_client = keep_client2
  45         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  46         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  47
  48         stubs.api = mock.MagicMock()
  49         stubs.api._rootDesc = get_rootDesc()
  50
  51         stubs.api.users().current().execute.return_value = {
  52             "uuid": stubs.fake_user_uuid,
  53         }
  54         stubs.api.collections().list().execute.return_value = {"items": []}
  55
  56         class CollectionExecute(object):
  57             def __init__(self, exe):
  58                 self.exe = exe
  59             def execute(self, num_retries=None):
  60                 return self.exe
  61
  62         def collection_createstub(created_collections, body, ensure_unique_name=None):
  63             mt = body["manifest_text"]
  64             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  65             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  66             created_collections[uuid] = {
  67                 "uuid": uuid,
  68                 "portable_data_hash": pdh,
  69                 "manifest_text": mt
  70             }
  71             return CollectionExecute(created_collections[uuid])
  72
  73         def collection_getstub(created_collections, uuid):
  74             for v in created_collections.itervalues():
  75                 if uuid in (v["uuid"], v["portable_data_hash"]):
  76                     return CollectionExecute(v)
  77
  78         created_collections = {}
  79         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  80         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 102                 'min_ram_mb_per_node': 1024
 103             },
 104             'script_parameters': {
 105                 'x': {
 106                     'basename': 'blorp.txt',
 107                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 108                     'class': 'File'
 109                 },
 110                 'y': {
 111                     'basename': '99999999999999999999999999999998+99',
 112                     'location': 'keep:99999999999999999999999999999998+99',
 113                     'class': 'Directory'
 114                 },
 115                 'z': {
 116                     'basename': 'anonymous',
 117                     "listing": [{
 118                         "basename": "renamed.txt",
 119                         "class": "File",
 120                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 121                     }],
 122                     'class': 'Directory'
 123                 },
 124                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 125             },
 126             'repository': 'arvados',
 127             'script_version': 'master',
 128             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 129             'script': 'cwl-runner'
 130         }
 131         stubs.pipeline_component = stubs.expect_job_spec.copy()
 132         stubs.expect_pipeline_instance = {
 133             'name': 'submit_wf.cwl',
 134             'state': 'RunningOnServer',
 135             'owner_uuid': None,
 136             "components": {
 137                 "cwl-runner": {
 138                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 139                     'script_parameters': {
 140                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 141                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
 142                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 143                               'listing': [
 144                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 145                               ]}},
 146                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 147                         'arv:enable_reuse': True,
 148                         'arv:on_error': 'continue'
 149                     },
 150                     'repository': 'arvados',
 151                     'script_version': 'master',
 152                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 153                     'script': 'cwl-runner',
 154                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 155                 }
 156             }
 157         }
 158         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 159         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 160         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 161         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 162         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 163             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 164             "state": "Queued"
 165         }
 166         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 167         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 168
 169         with open("tests/wf/submit_wf_packed.cwl") as f:
 170             expect_packed_workflow = yaml.round_trip_load(f)
 171
 172         stubs.expect_container_spec = {
 173             'priority': 1,
 174             'mounts': {
 175                 '/var/spool/cwl': {
 176                     'writable': True,
 177                     'kind': 'collection'
 178                 },
 179                 '/var/lib/cwl/workflow.json': {
 180                     'content': expect_packed_workflow,
 181                     'kind': 'json'
 182                 },
 183                 'stdout': {
 184                     'path': '/var/spool/cwl/cwl.output.json',
 185                     'kind': 'file'
 186                 },
 187                 '/var/lib/cwl/cwl.input.json': {
 188                     'kind': 'json',
 189                     'content': {
 190                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 191                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
 192                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 193                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 194                         ]}
 195                     },
 196                     'kind': 'json'
 197                 }
 198             },
 199             'state': 'Committed',
 200             'owner_uuid': None,
 201             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 202                         '--enable-reuse', '--on-error=continue',
 203                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 204             'name': 'submit_wf.cwl',
 205             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 206             'output_path': '/var/spool/cwl',
 207             'cwd': '/var/spool/cwl',
 208             'runtime_constraints': {
 209                 'API': True,
 210                 'vcpus': 1,
 211                 'ram': 1024*1024*1024
 212             },
 213             "properties": {}
 214         }
 215
 216         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 217         stubs.api.workflows().create().execute.return_value = {
 218             "uuid": stubs.expect_workflow_uuid,
 219         }
 220         def update_mock(**kwargs):
 221             stubs.updated_uuid = kwargs.get('uuid')
 222             return mock.DEFAULT
 223         stubs.api.workflows().update.side_effect = update_mock
 224         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 225             "uuid": stubs.updated_uuid,
 226         }
 227
 228         return func(self, stubs, *args, **kwargs)
 229     return wrapped
 230
 231
 232 class TestSubmit(unittest.TestCase):
 233     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 234     @mock.patch("time.sleep")
 235     @stubs
 236     def test_submit(self, stubs, tm, arvdock):
 237         capture_stdout = cStringIO.StringIO()
 238         exited = arvados_cwl.main(
 239             ["--submit", "--no-wait", "--api=jobs", "--debug",
 240              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 241             capture_stdout, sys.stderr, api_client=stubs.api)
 242         self.assertEqual(exited, 0)
 243
 244         stubs.api.collections().create.assert_has_calls([
 245             mock.call(body=JsonDiffMatcher({
 246                 'manifest_text':
 247                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 248                 'replication_desired': None,
 249                 'name': 'submit_tool.cwl dependencies',
 250             }), ensure_unique_name=True),
 251             mock.call(body=JsonDiffMatcher({
 252                 'manifest_text':
 253                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 254                 'replication_desired': None,
 255                 'name': 'submit_wf.cwl input',
 256             }), ensure_unique_name=True),
 257             mock.call(body=JsonDiffMatcher({
 258                 'manifest_text':
 259                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 260                 'replication_desired': None,
 261                 'name': 'submit_wf.cwl',
 262             }), ensure_unique_name=True)        ])
 263
 264         arvdock.assert_has_calls([
 265             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 266             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 267         ])
 268
 269         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 270         stubs.api.pipeline_instances().create.assert_called_with(
 271             body=JsonDiffMatcher(expect_pipeline))
 272         self.assertEqual(capture_stdout.getvalue(),
 273                          stubs.expect_pipeline_uuid + '\n')
 274
 275
 276     @mock.patch("time.sleep")
 277     @stubs
 278     def test_submit_no_reuse(self, stubs, tm):
 279         capture_stdout = cStringIO.StringIO()
 280         exited = arvados_cwl.main(
 281             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 282              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 283             capture_stdout, sys.stderr, api_client=stubs.api)
 284         self.assertEqual(exited, 0)
 285
 286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 287         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 288
 289         stubs.api.pipeline_instances().create.assert_called_with(
 290             body=JsonDiffMatcher(expect_pipeline))
 291         self.assertEqual(capture_stdout.getvalue(),
 292                          stubs.expect_pipeline_uuid + '\n')
 293
 294     @mock.patch("time.sleep")
 295     @stubs
 296     def test_submit_on_error(self, stubs, tm):
 297         capture_stdout = cStringIO.StringIO()
 298         exited = arvados_cwl.main(
 299             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 301             capture_stdout, sys.stderr, api_client=stubs.api)
 302         self.assertEqual(exited, 0)
 303
 304         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 305         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 306
 307         stubs.api.pipeline_instances().create.assert_called_with(
 308             body=JsonDiffMatcher(expect_pipeline))
 309         self.assertEqual(capture_stdout.getvalue(),
 310                          stubs.expect_pipeline_uuid + '\n')
 311
 312
 313     @mock.patch("time.sleep")
 314     @stubs
 315     def test_submit_runner_ram(self, stubs, tm):
 316         capture_stdout = cStringIO.StringIO()
 317         exited = arvados_cwl.main(
 318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 0)
 322
 323         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 324         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331
 332     @mock.patch("time.sleep")
 333     @stubs
 334     def test_submit_invalid_runner_ram(self, stubs, tm):
 335         capture_stdout = cStringIO.StringIO()
 336         exited = arvados_cwl.main(
 337             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 338              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 339             capture_stdout, sys.stderr, api_client=stubs.api)
 340         self.assertEqual(exited, 1)
 341
 342     @mock.patch("time.sleep")
 343     @stubs
 344     def test_submit_output_name(self, stubs, tm):
 345         output_name = "test_output_name"
 346
 347         capture_stdout = cStringIO.StringIO()
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             capture_stdout, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 0)
 353
 354         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 355         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 356
 357         stubs.api.pipeline_instances().create.assert_called_with(
 358             body=JsonDiffMatcher(expect_pipeline))
 359         self.assertEqual(capture_stdout.getvalue(),
 360                          stubs.expect_pipeline_uuid + '\n')
 361
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_pipeline_name(self, stubs, tm):
 366         capture_stdout = cStringIO.StringIO()
 367         exited = arvados_cwl.main(
 368             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 369              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 370             capture_stdout, sys.stderr, api_client=stubs.api)
 371         self.assertEqual(exited, 0)
 372
 373         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 374         expect_pipeline["name"] = "hello job 123"
 375
 376         stubs.api.pipeline_instances().create.assert_called_with(
 377             body=JsonDiffMatcher(expect_pipeline))
 378         self.assertEqual(capture_stdout.getvalue(),
 379                          stubs.expect_pipeline_uuid + '\n')
 380
 381     @mock.patch("time.sleep")
 382     @stubs
 383     def test_submit_output_tags(self, stubs, tm):
 384         output_tags = "tag0,tag1,tag2"
 385
 386         capture_stdout = cStringIO.StringIO()
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 390             capture_stdout, sys.stderr, api_client=stubs.api)
 391         self.assertEqual(exited, 0)
 392
 393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 395
 396         stubs.api.pipeline_instances().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_pipeline))
 398         self.assertEqual(capture_stdout.getvalue(),
 399                          stubs.expect_pipeline_uuid + '\n')
 400
 401     @mock.patch("time.sleep")
 402     @stubs
 403     def test_submit_with_project_uuid(self, stubs, tm):
 404         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 405
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait",
 408              "--project-uuid", project_uuid,
 409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             sys.stdout, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 414         expect_pipeline["owner_uuid"] = project_uuid
 415         stubs.api.pipeline_instances().create.assert_called_with(
 416             body=JsonDiffMatcher(expect_pipeline))
 417
 418     @stubs
 419     def test_submit_container(self, stubs):
 420         capture_stdout = cStringIO.StringIO()
 421         try:
 422             exited = arvados_cwl.main(
 423                 ["--submit", "--no-wait", "--api=containers", "--debug",
 424                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 426             self.assertEqual(exited, 0)
 427         except:
 428             logging.exception("")
 429
 430         stubs.api.collections().create.assert_has_calls([
 431             mock.call(body=JsonDiffMatcher({
 432                 'manifest_text':
 433                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 434                 'replication_desired': None,
 435                 'name': 'submit_tool.cwl dependencies',
 436             }), ensure_unique_name=True),
 437             mock.call(body=JsonDiffMatcher({
 438                 'manifest_text':
 439                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 440                 'replication_desired': None,
 441                 'name': 'submit_wf.cwl input',
 442             }), ensure_unique_name=True)])
 443
 444         expect_container = copy.deepcopy(stubs.expect_container_spec)
 445         stubs.api.container_requests().create.assert_called_with(
 446             body=JsonDiffMatcher(expect_container))
 447         self.assertEqual(capture_stdout.getvalue(),
 448                          stubs.expect_container_request_uuid + '\n')
 449
 450     @stubs
 451     def test_submit_container_no_reuse(self, stubs):
 452         capture_stdout = cStringIO.StringIO()
 453         try:
 454             exited = arvados_cwl.main(
 455                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 456                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 457                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 458             self.assertEqual(exited, 0)
 459         except:
 460             logging.exception("")
 461
 462         expect_container = copy.deepcopy(stubs.expect_container_spec)
 463         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 464                                                   '--disable-reuse', '--on-error=continue',
 465                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 466
 467         stubs.api.container_requests().create.assert_called_with(
 468             body=JsonDiffMatcher(expect_container))
 469         self.assertEqual(capture_stdout.getvalue(),
 470                          stubs.expect_container_request_uuid + '\n')
 471
 472
 473     @stubs
 474     def test_submit_container_on_error(self, stubs):
 475         capture_stdout = cStringIO.StringIO()
 476         try:
 477             exited = arvados_cwl.main(
 478                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 479                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 480                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 481             self.assertEqual(exited, 0)
 482         except:
 483             logging.exception("")
 484
 485         expect_container = copy.deepcopy(stubs.expect_container_spec)
 486         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 487                                                   '--enable-reuse', '--on-error=stop',
 488                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 489
 490         stubs.api.container_requests().create.assert_called_with(
 491             body=JsonDiffMatcher(expect_container))
 492         self.assertEqual(capture_stdout.getvalue(),
 493                          stubs.expect_container_request_uuid + '\n')
 494
 495     @stubs
 496     def test_submit_container_output_name(self, stubs):
 497         output_name = "test_output_name"
 498
 499         capture_stdout = cStringIO.StringIO()
 500         try:
 501             exited = arvados_cwl.main(
 502                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 503                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505             self.assertEqual(exited, 0)
 506         except:
 507             logging.exception("")
 508
 509         expect_container = copy.deepcopy(stubs.expect_container_spec)
 510         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 511                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 512                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 513         expect_container["output_name"] = output_name
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519
 520     @stubs
 521     def test_submit_container_output_tags(self, stubs):
 522         output_tags = "tag0,tag1,tag2"
 523
 524         capture_stdout = cStringIO.StringIO()
 525         try:
 526             exited = arvados_cwl.main(
 527                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 528                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 529                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 530             self.assertEqual(exited, 0)
 531         except:
 532             logging.exception("")
 533
 534         expect_container = copy.deepcopy(stubs.expect_container_spec)
 535         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 536                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 537                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 538
 539         stubs.api.container_requests().create.assert_called_with(
 540             body=JsonDiffMatcher(expect_container))
 541         self.assertEqual(capture_stdout.getvalue(),
 542                          stubs.expect_container_request_uuid + '\n')
 543
 544     @stubs
 545     def test_submit_container_runner_ram(self, stubs):
 546         capture_stdout = cStringIO.StringIO()
 547         try:
 548             exited = arvados_cwl.main(
 549                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 550                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 551                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 552             self.assertEqual(exited, 0)
 553         except:
 554             logging.exception("")
 555
 556         expect_container = copy.deepcopy(stubs.expect_container_spec)
 557         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 558
 559         stubs.api.container_requests().create.assert_called_with(
 560             body=JsonDiffMatcher(expect_container))
 561         self.assertEqual(capture_stdout.getvalue(),
 562                          stubs.expect_container_request_uuid + '\n')
 563
 564     @mock.patch("arvados.collection.CollectionReader")
 565     @mock.patch("time.sleep")
 566     @stubs
 567     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 568         capture_stdout = cStringIO.StringIO()
 569         exited = arvados_cwl.main(
 570             ["--submit", "--no-wait", "--api=containers", "--debug",
 571              "tests/wf/submit_keepref_wf.cwl"],
 572             capture_stdout, sys.stderr, api_client=stubs.api)
 573         self.assertEqual(exited, 0)
 574
 575
 576     @mock.patch("arvados.collection.CollectionReader")
 577     @mock.patch("time.sleep")
 578     @stubs
 579     def test_submit_keepref(self, stubs, tm, reader):
 580         capture_stdout = cStringIO.StringIO()
 581
 582         with open("tests/wf/expect_arvworkflow.cwl") as f:
 583             reader().open().__enter__().read.return_value = f.read()
 584
 585         exited = arvados_cwl.main(
 586             ["--submit", "--no-wait", "--api=containers", "--debug",
 587              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 588             capture_stdout, sys.stderr, api_client=stubs.api)
 589         self.assertEqual(exited, 0)
 590
 591         expect_container = {
 592             'priority': 1,
 593             'mounts': {
 594                 '/var/spool/cwl': {
 595                     'writable': True,
 596                     'kind': 'collection'
 597                 },
 598                 'stdout': {
 599                     'path': '/var/spool/cwl/cwl.output.json',
 600                     'kind': 'file'
 601                 },
 602                 '/var/lib/cwl/workflow': {
 603                     'portable_data_hash': '99999999999999999999999999999994+99',
 604                     'kind': 'collection'
 605                 },
 606                 '/var/lib/cwl/cwl.input.json': {
 607                     'content': {
 608                         'x': 'XxX'
 609                     },
 610                     'kind': 'json'
 611                 }
 612             }, 'state': 'Committed',
 613             'owner_uuid': None,
 614             'output_path': '/var/spool/cwl',
 615             'name': 'expect_arvworkflow.cwl#main',
 616             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 617             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 618                         '--enable-reuse', '--on-error=continue',
 619                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 620             'cwd': '/var/spool/cwl',
 621             'runtime_constraints': {
 622                 'API': True,
 623                 'vcpus': 1,
 624                 'ram': 1073741824
 625             },
 626             "properties": {}
 627         }
 628
 629         stubs.api.container_requests().create.assert_called_with(
 630             body=JsonDiffMatcher(expect_container))
 631         self.assertEqual(capture_stdout.getvalue(),
 632                          stubs.expect_container_request_uuid + '\n')
 633
 634
 635     @mock.patch("arvados.collection.CollectionReader")
 636     @mock.patch("time.sleep")
 637     @stubs
 638     def test_submit_jobs_keepref(self, stubs, tm, reader):
 639         capture_stdout = cStringIO.StringIO()
 640
 641         with open("tests/wf/expect_arvworkflow.cwl") as f:
 642             reader().open().__enter__().read.return_value = f.read()
 643
 644         exited = arvados_cwl.main(
 645             ["--submit", "--no-wait", "--api=jobs", "--debug",
 646              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 647             capture_stdout, sys.stderr, api_client=stubs.api)
 648         self.assertEqual(exited, 0)
 649
 650         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 651         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 652         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 653         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 654         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 655         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 656         stubs.api.pipeline_instances().create.assert_called_with(
 657             body=JsonDiffMatcher(expect_pipeline))
 658
 659     @mock.patch("time.sleep")
 660     @stubs
 661     def test_submit_arvworkflow(self, stubs, tm):
 662         capture_stdout = cStringIO.StringIO()
 663
 664         with open("tests/wf/expect_arvworkflow.cwl") as f:
 665             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 666
 667         exited = arvados_cwl.main(
 668             ["--submit", "--no-wait", "--api=containers", "--debug",
 669              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 670             capture_stdout, sys.stderr, api_client=stubs.api)
 671         self.assertEqual(exited, 0)
 672
 673         expect_container = {
 674             'priority': 1,
 675             'mounts': {
 676                 '/var/spool/cwl': {
 677                     'writable': True,
 678                     'kind': 'collection'
 679                 },
 680                 'stdout': {
 681                     'path': '/var/spool/cwl/cwl.output.json',
 682                     'kind': 'file'
 683                 },
 684                 '/var/lib/cwl/workflow.json': {
 685                     'kind': 'json',
 686                     'content': {
 687                         'cwlVersion': 'v1.0',
 688                         '$graph': [
 689                             {
 690                                 'id': '#main',
 691                                 'inputs': [
 692                                     {'type': 'string', 'id': '#main/x'}
 693                                 ],
 694                                 'steps': [
 695                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 696                                      'run': '#submit_tool.cwl',
 697                                      'id': '#main/step1',
 698                                      'out': []}
 699                                 ],
 700                                 'class': 'Workflow',
 701                                 'outputs': []
 702                             },
 703                             {
 704                                 'inputs': [
 705                                     {
 706                                         'inputBinding': {'position': 1},
 707                                         'type': 'string',
 708                                         'id': '#submit_tool.cwl/x'}
 709                                 ],
 710                                 'requirements': [
 711                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 712                                 ],
 713                                 'id': '#submit_tool.cwl',
 714                                 'outputs': [],
 715                                 'baseCommand': 'cat',
 716                                 'class': 'CommandLineTool'
 717                             }
 718                         ]
 719                     }
 720                 },
 721                 '/var/lib/cwl/cwl.input.json': {
 722                     'content': {
 723                         'x': 'XxX'
 724                     },
 725                     'kind': 'json'
 726                 }
 727             }, 'state': 'Committed',
 728             'owner_uuid': None,
 729             'output_path': '/var/spool/cwl',
 730             'name': 'a test workflow',
 731             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 732             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 733                         '--enable-reuse', '--on-error=continue',
 734                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 735             'cwd': '/var/spool/cwl',
 736             'runtime_constraints': {
 737                 'API': True,
 738                 'vcpus': 1,
 739                 'ram': 1073741824
 740             },
 741             "properties": {
 742                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 743             }
 744         }
 745
 746         stubs.api.container_requests().create.assert_called_with(
 747             body=JsonDiffMatcher(expect_container))
 748         self.assertEqual(capture_stdout.getvalue(),
 749                          stubs.expect_container_request_uuid + '\n')
 750
 751
 752     @stubs
 753     def test_submit_container_name(self, stubs):
 754         capture_stdout = cStringIO.StringIO()
 755         try:
 756             exited = arvados_cwl.main(
 757                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 758                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 759                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 760             self.assertEqual(exited, 0)
 761         except:
 762             logging.exception("")
 763
 764         expect_container = copy.deepcopy(stubs.expect_container_spec)
 765         expect_container["name"] = "hello container 123"
 766
 767         stubs.api.container_requests().create.assert_called_with(
 768             body=JsonDiffMatcher(expect_container))
 769         self.assertEqual(capture_stdout.getvalue(),
 770                          stubs.expect_container_request_uuid + '\n')
 771
 772
 773     @stubs
 774     def test_submit_job_runner_image(self, stubs):
 775         capture_stdout = cStringIO.StringIO()
 776         try:
 777             exited = arvados_cwl.main(
 778                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 779                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 780                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 781             self.assertEqual(exited, 0)
 782         except:
 783             logging.exception("")
 784
 785         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 786
 787         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 788         stubs.api.pipeline_instances().create.assert_called_with(
 789             body=JsonDiffMatcher(expect_pipeline))
 790         self.assertEqual(capture_stdout.getvalue(),
 791                          stubs.expect_pipeline_uuid + '\n')
 792
 793     @stubs
 794     def test_submit_container_runner_image(self, stubs):
 795         capture_stdout = cStringIO.StringIO()
 796         try:
 797             exited = arvados_cwl.main(
 798                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 799                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 800                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 801             self.assertEqual(exited, 0)
 802         except:
 803             logging.exception("")
 804
 805         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 806
 807         expect_container = copy.deepcopy(stubs.expect_container_spec)
 808         stubs.api.container_requests().create.assert_called_with(
 809             body=JsonDiffMatcher(expect_container))
 810         self.assertEqual(capture_stdout.getvalue(),
 811                          stubs.expect_container_request_uuid + '\n')
 812
 813
 814     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 815     @mock.patch("cwltool.docker.get_image")
 816     @mock.patch("arvados.api")
 817     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 818         arvrunner = mock.MagicMock()
 819         arvrunner.project_uuid = ""
 820         api.return_value = mock.MagicMock()
 821         arvrunner.api = api.return_value
 822         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 823                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 824                                                                         "link_class": "docker_image_repo+tag",
 825                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 826                                                                         "owner_uuid": "",
 827                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 828                                                             {"items": [{"created_at": "",
 829                                                                         "head_uuid": "",
 830                                                                         "link_class": "docker_image_hash",
 831                                                                         "name": "123456",
 832                                                                         "owner_uuid": "",
 833                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 834         )
 835         find_one_image_hash.return_value = "123456"
 836
 837         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 838                                                                               "owner_uuid": "",
 839                                                                               "manifest_text": "",
 840                                                                               "properties": ""
 841                                                                           }], "items_available": 1, "offset": 0},)
 842         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 843         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 844                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 845
 846 class TestCreateTemplate(unittest.TestCase):
 847     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 848
 849     def _adjust_script_params(self, expect_component):
 850         expect_component['script_parameters']['x'] = {
 851             'dataclass': 'File',
 852             'required': True,
 853             'type': 'File',
 854             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 855         }
 856         expect_component['script_parameters']['y'] = {
 857             'dataclass': 'Collection',
 858             'required': True,
 859             'type': 'Directory',
 860             'value': '99999999999999999999999999999998+99',
 861         }
 862         expect_component['script_parameters']['z'] = {
 863             'dataclass': 'Collection',
 864             'required': True,
 865             'type': 'Directory',
 866         }
 867
 868     @stubs
 869     def test_create(self, stubs):
 870         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 871
 872         capture_stdout = cStringIO.StringIO()
 873
 874         exited = arvados_cwl.main(
 875             ["--create-workflow", "--debug",
 876              "--api=jobs",
 877              "--project-uuid", project_uuid,
 878              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 879             capture_stdout, sys.stderr, api_client=stubs.api)
 880         self.assertEqual(exited, 0)
 881
 882         stubs.api.pipeline_instances().create.refute_called()
 883         stubs.api.jobs().create.refute_called()
 884
 885         expect_component = copy.deepcopy(stubs.expect_job_spec)
 886         self._adjust_script_params(expect_component)
 887         expect_template = {
 888             "components": {
 889                 "submit_wf.cwl": expect_component,
 890             },
 891             "name": "submit_wf.cwl",
 892             "owner_uuid": project_uuid,
 893         }
 894         stubs.api.pipeline_templates().create.assert_called_with(
 895             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 896
 897         self.assertEqual(capture_stdout.getvalue(),
 898                          stubs.expect_pipeline_template_uuid + '\n')
 899
 900
 901     @stubs
 902     def test_create_name(self, stubs):
 903         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 904
 905         capture_stdout = cStringIO.StringIO()
 906
 907         exited = arvados_cwl.main(
 908             ["--create-workflow", "--debug",
 909              "--project-uuid", project_uuid,
 910              "--api=jobs",
 911              "--name", "testing 123",
 912              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 913             capture_stdout, sys.stderr, api_client=stubs.api)
 914         self.assertEqual(exited, 0)
 915
 916         stubs.api.pipeline_instances().create.refute_called()
 917         stubs.api.jobs().create.refute_called()
 918
 919         expect_component = copy.deepcopy(stubs.expect_job_spec)
 920         self._adjust_script_params(expect_component)
 921         expect_template = {
 922             "components": {
 923                 "testing 123": expect_component,
 924             },
 925             "name": "testing 123",
 926             "owner_uuid": project_uuid,
 927         }
 928         stubs.api.pipeline_templates().create.assert_called_with(
 929             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 930
 931         self.assertEqual(capture_stdout.getvalue(),
 932                          stubs.expect_pipeline_template_uuid + '\n')
 933
 934
 935     @stubs
 936     def test_update_name(self, stubs):
 937         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 938
 939         capture_stdout = cStringIO.StringIO()
 940
 941         exited = arvados_cwl.main(
 942             ["--update-workflow", self.existing_template_uuid,
 943              "--debug",
 944              "--project-uuid", project_uuid,
 945              "--api=jobs",
 946              "--name", "testing 123",
 947              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 948             capture_stdout, sys.stderr, api_client=stubs.api)
 949         self.assertEqual(exited, 0)
 950
 951         stubs.api.pipeline_instances().create.refute_called()
 952         stubs.api.jobs().create.refute_called()
 953
 954         expect_component = copy.deepcopy(stubs.expect_job_spec)
 955         self._adjust_script_params(expect_component)
 956         expect_template = {
 957             "components": {
 958                 "testing 123": expect_component,
 959             },
 960             "name": "testing 123",
 961             "owner_uuid": project_uuid,
 962         }
 963         stubs.api.pipeline_templates().create.refute_called()
 964         stubs.api.pipeline_templates().update.assert_called_with(
 965             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 966
 967         self.assertEqual(capture_stdout.getvalue(),
 968                          self.existing_template_uuid + '\n')
 969
 970
 971 class TestCreateWorkflow(unittest.TestCase):
 972     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 973     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 974
 975     @stubs
 976     def test_create(self, stubs):
 977         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 978
 979         capture_stdout = cStringIO.StringIO()
 980
 981         exited = arvados_cwl.main(
 982             ["--create-workflow", "--debug",
 983              "--api=containers",
 984              "--project-uuid", project_uuid,
 985              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 986             capture_stdout, sys.stderr, api_client=stubs.api)
 987         self.assertEqual(exited, 0)
 988
 989         stubs.api.pipeline_templates().create.refute_called()
 990         stubs.api.container_requests().create.refute_called()
 991
 992         body = {
 993             "workflow": {
 994                 "owner_uuid": project_uuid,
 995                 "name": "submit_wf.cwl",
 996                 "description": "",
 997                 "definition": self.expect_workflow,
 998             }
 999         }
1000         stubs.api.workflows().create.assert_called_with(
1001             body=JsonDiffMatcher(body))
1002
1003         self.assertEqual(capture_stdout.getvalue(),
1004                          stubs.expect_workflow_uuid + '\n')
1005
1006
1007     @stubs
1008     def test_create_name(self, stubs):
1009         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1010
1011         capture_stdout = cStringIO.StringIO()
1012
1013         exited = arvados_cwl.main(
1014             ["--create-workflow", "--debug",
1015              "--api=containers",
1016              "--project-uuid", project_uuid,
1017              "--name", "testing 123",
1018              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1019             capture_stdout, sys.stderr, api_client=stubs.api)
1020         self.assertEqual(exited, 0)
1021
1022         stubs.api.pipeline_templates().create.refute_called()
1023         stubs.api.container_requests().create.refute_called()
1024
1025         body = {
1026             "workflow": {
1027                 "owner_uuid": project_uuid,
1028                 "name": "testing 123",
1029                 "description": "",
1030                 "definition": self.expect_workflow,
1031             }
1032         }
1033         stubs.api.workflows().create.assert_called_with(
1034             body=JsonDiffMatcher(body))
1035
1036         self.assertEqual(capture_stdout.getvalue(),
1037                          stubs.expect_workflow_uuid + '\n')
1038
1039     @stubs
1040     def test_incompatible_api(self, stubs):
1041         capture_stderr = cStringIO.StringIO()
1042         logging.getLogger('arvados.cwl-runner').addHandler(
1043             logging.StreamHandler(capture_stderr))
1044
1045         exited = arvados_cwl.main(
1046             ["--update-workflow", self.existing_workflow_uuid,
1047              "--api=jobs",
1048              "--debug",
1049              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1050             sys.stderr, sys.stderr, api_client=stubs.api)
1051         self.assertEqual(exited, 1)
1052         self.assertRegexpMatches(
1053             capture_stderr.getvalue(),
1054             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1055
1056     @stubs
1057     def test_update(self, stubs):
1058         capture_stdout = cStringIO.StringIO()
1059
1060         exited = arvados_cwl.main(
1061             ["--update-workflow", self.existing_workflow_uuid,
1062              "--debug",
1063              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1064             capture_stdout, sys.stderr, api_client=stubs.api)
1065         self.assertEqual(exited, 0)
1066
1067         body = {
1068             "workflow": {
1069                 "name": "submit_wf.cwl",
1070                 "description": "",
1071                 "definition": self.expect_workflow,
1072             }
1073         }
1074         stubs.api.workflows().update.assert_called_with(
1075             uuid=self.existing_workflow_uuid,
1076             body=JsonDiffMatcher(body))
1077         self.assertEqual(capture_stdout.getvalue(),
1078                          self.existing_workflow_uuid + '\n')
1079
1080
1081     @stubs
1082     def test_update_name(self, stubs):
1083         capture_stdout = cStringIO.StringIO()
1084
1085         exited = arvados_cwl.main(
1086             ["--update-workflow", self.existing_workflow_uuid,
1087              "--debug", "--name", "testing 123",
1088              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1089             capture_stdout, sys.stderr, api_client=stubs.api)
1090         self.assertEqual(exited, 0)
1091
1092         body = {
1093             "workflow": {
1094                 "name": "testing 123",
1095                 "description": "",
1096                 "definition": self.expect_workflow,
1097             }
1098         }
1099         stubs.api.workflows().update.assert_called_with(
1100             uuid=self.existing_workflow_uuid,
1101             body=JsonDiffMatcher(body))
1102         self.assertEqual(capture_stdout.getvalue(),
1103                          self.existing_workflow_uuid + '\n')
1104
1105
1106     @stubs
1107     def test_create_collection_per_tool(self, stubs):
1108         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1109
1110         capture_stdout = cStringIO.StringIO()
1111
1112         exited = arvados_cwl.main(
1113             ["--create-workflow", "--debug",
1114              "--api=containers",
1115              "--project-uuid", project_uuid,
1116              "tests/collection_per_tool/collection_per_tool.cwl"],
1117             capture_stdout, sys.stderr, api_client=stubs.api)
1118         self.assertEqual(exited, 0)
1119
1120         expect_workflow = open("tests/collection_per_tool/collection_per_tool_packed.cwl").read()
1121
1122         body = {
1123             "workflow": {
1124                 "owner_uuid": project_uuid,
1125                 "name": "collection_per_tool.cwl",
1126                 "description": "",
1127                 "definition": expect_workflow,
1128             }
1129         }
1130         stubs.api.workflows().create.assert_called_with(
1131             body=JsonDiffMatcher(body))
1132
1133         self.assertEqual(capture_stdout.getvalue(),
1134                          stubs.expect_workflow_uuid + '\n')
1135
1136 class TestTemplateInputs(unittest.TestCase):
1137     expect_template = {
1138         "components": {
1139             "inputs_test.cwl": {
1140                 'runtime_constraints': {
1141                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1142                     'min_ram_mb_per_node': 1024
1143                 },
1144                 'script_parameters': {
1145                     'cwl:tool':
1146                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1147                     'optionalFloatInput': None,
1148                     'fileInput': {
1149                         'type': 'File',
1150                         'dataclass': 'File',
1151                         'required': True,
1152                         'title': "It's a file; we expect to find some characters in it.",
1153                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1154                     },
1155                     'floatInput': {
1156                         'type': 'float',
1157                         'dataclass': 'number',
1158                         'required': True,
1159                         'title': 'Floats like a duck',
1160                         'default': 0.1,
1161                         'value': 0.1,
1162                     },
1163                     'optionalFloatInput': {
1164                         'type': ['null', 'float'],
1165                         'dataclass': 'number',
1166                         'required': False,
1167                     },
1168                     'boolInput': {
1169                         'type': 'boolean',
1170                         'dataclass': 'boolean',
1171                         'required': True,
1172                         'title': 'True or false?',
1173                     },
1174                 },
1175                 'repository': 'arvados',
1176                 'script_version': 'master',
1177                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1178                 'script': 'cwl-runner',
1179             },
1180         },
1181         "name": "inputs_test.cwl",
1182     }
1183
1184     @stubs
1185     def test_inputs_empty(self, stubs):
1186         exited = arvados_cwl.main(
1187             ["--create-template",
1188              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1189             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1190         self.assertEqual(exited, 0)
1191
1192         stubs.api.pipeline_templates().create.assert_called_with(
1193             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1194
1195     @stubs
1196     def test_inputs(self, stubs):
1197         exited = arvados_cwl.main(
1198             ["--create-template",
1199              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1200             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1201         self.assertEqual(exited, 0)
1202
1203         expect_template = copy.deepcopy(self.expect_template)
1204         params = expect_template[
1205             "components"]["inputs_test.cwl"]["script_parameters"]
1206         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1207         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1208         params["floatInput"]["value"] = 1.234
1209         params["boolInput"]["value"] = True
1210
1211         stubs.api.pipeline_templates().create.assert_called_with(
1212             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)