sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 import ruamel.yaml as yaml
  21
  22 _rootDesc = None
  23
  24 def stubs(func):
  25     @functools.wraps(func)
  26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  27     @mock.patch("arvados.collection.KeepClient")
  28     @mock.patch("arvados.keep.KeepClient")
  29     @mock.patch("arvados.events.subscribe")
  30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  31         class Stubs:
  32             pass
  33         stubs = Stubs()
  34         stubs.events = events
  35         stubs.keepdocker = keepdocker
  36
  37
  38         def putstub(p, **kwargs):
  39             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  40         keep_client1().put.side_effect = putstub
  41         keep_client1.put.side_effect = putstub
  42         keep_client2().put.side_effect = putstub
  43         keep_client2.put.side_effect = putstub
  44
  45         stubs.keep_client = keep_client2
  46         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  47         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  48
  49         stubs.api = mock.MagicMock()
  50         stubs.api._rootDesc = get_rootDesc()
  51
  52         stubs.api.users().current().execute.return_value = {
  53             "uuid": stubs.fake_user_uuid,
  54         }
  55         stubs.api.collections().list().execute.return_value = {"items": []}
  56         stubs.api.collections().create().execute.side_effect = ({
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  58             "portable_data_hash": "99999999999999999999999999999991+99",
  59             "manifest_text": ""
  60         }, {
  61             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  62             "portable_data_hash": "99999999999999999999999999999992+99",
  63             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  64         },
  65         {
  66             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  67             "portable_data_hash": "99999999999999999999999999999994+99",
  68             "manifest_text": ""
  69         },
  70         {
  71             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  72             "portable_data_hash": "99999999999999999999999999999995+99",
  73             "manifest_text": ""
  74         },
  75         {
  76             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  77             "portable_data_hash": "99999999999999999999999999999996+99",
  78             "manifest_text": ""
  79         }
  80         )
  81         stubs.api.collections().get().execute.return_value = {
  82             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  83
  84         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  85         stubs.api.jobs().create().execute.return_value = {
  86             "uuid": stubs.expect_job_uuid,
  87             "state": "Queued",
  88         }
  89
  90         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  91         stubs.api.container_requests().create().execute.return_value = {
  92             "uuid": stubs.expect_container_request_uuid,
  93             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  94             "state": "Queued"
  95         }
  96
  97         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  98         stubs.api.pipeline_templates().create().execute.return_value = {
  99             "uuid": stubs.expect_pipeline_template_uuid,
 100         }
 101         stubs.expect_job_spec = {
 102             'runtime_constraints': {
 103                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 104                 'min_ram_mb_per_node': 1024
 105             },
 106             'script_parameters': {
 107                 'x': {
 108                     'basename': 'blorp.txt',
 109                     'location': 'keep:99999999999999999999999999999992+99/blorp.txt',
 110                     'class': 'File'
 111                 },
 112                 'y': {
 113                     'basename': '99999999999999999999999999999998+99',
 114                     'location': 'keep:99999999999999999999999999999998+99',
 115                     'class': 'Directory'
 116                 },
 117                 'z': {
 118                     'basename': 'anonymous',
 119                     "listing": [{
 120                         "basename": "renamed.txt",
 121                         "class": "File",
 122                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 123                     }],
 124                     'class': 'Directory'
 125                 },
 126                 'cwl:tool':
 127                 '99999999999999999999999999999994+99/workflow.cwl#main'
 128             },
 129             'repository': 'arvados',
 130             'script_version': 'master',
 131             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 132             'script': 'cwl-runner'
 133         }
 134         stubs.pipeline_component = stubs.expect_job_spec.copy()
 135         stubs.expect_pipeline_instance = {
 136             'name': 'submit_wf.cwl',
 137             'state': 'RunningOnServer',
 138             'owner_uuid': None,
 139             "components": {
 140                 "cwl-runner": {
 141                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 142                     'script_parameters': {
 143                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 144                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999992+99/blorp.txt'}},
 145                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 146                               'listing': [
 147                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 148                               ]}},
 149                         'cwl:tool': '99999999999999999999999999999994+99/workflow.cwl#main',
 150                         'arv:enable_reuse': True,
 151                         'arv:on_error': 'continue'
 152                     },
 153                     'repository': 'arvados',
 154                     'script_version': 'master',
 155                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 156                     'script': 'cwl-runner',
 157                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 158                 }
 159             }
 160         }
 161         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 162         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 163         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 164         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 165         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 166             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 167             "state": "Queued"
 168         }
 169         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 170         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 171
 172         with open("tests/wf/submit_wf_packed.cwl") as f:
 173             expect_packed_workflow = yaml.round_trip_load(f)
 174
 175         stubs.expect_container_spec = {
 176             'priority': 1,
 177             'mounts': {
 178                 '/var/spool/cwl': {
 179                     'writable': True,
 180                     'kind': 'collection'
 181                 },
 182                 '/var/lib/cwl/workflow.json': {
 183                     'content': expect_packed_workflow,
 184                     'kind': 'json'
 185                 },
 186                 'stdout': {
 187                     'path': '/var/spool/cwl/cwl.output.json',
 188                     'kind': 'file'
 189                 },
 190                 '/var/lib/cwl/cwl.input.json': {
 191                     'kind': 'json',
 192                     'content': {
 193                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 194                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999992+99/blorp.txt'},
 195                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 196                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 197                         ]}
 198                     },
 199                     'kind': 'json'
 200                 }
 201             },
 202             'state': 'Committed',
 203             'owner_uuid': None,
 204             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 205                         '--enable-reuse', '--on-error=continue',
 206                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 207             'name': 'submit_wf.cwl',
 208             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 209             'output_path': '/var/spool/cwl',
 210             'cwd': '/var/spool/cwl',
 211             'runtime_constraints': {
 212                 'API': True,
 213                 'vcpus': 1,
 214                 'ram': 1024*1024*1024
 215             },
 216             "properties": {}
 217         }
 218
 219         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 220         stubs.api.workflows().create().execute.return_value = {
 221             "uuid": stubs.expect_workflow_uuid,
 222         }
 223         def update_mock(**kwargs):
 224             stubs.updated_uuid = kwargs.get('uuid')
 225             return mock.DEFAULT
 226         stubs.api.workflows().update.side_effect = update_mock
 227         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 228             "uuid": stubs.updated_uuid,
 229         }
 230
 231         return func(self, stubs, *args, **kwargs)
 232     return wrapped
 233
 234
 235 class TestSubmit(unittest.TestCase):
 236     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 237     @mock.patch("time.sleep")
 238     @stubs
 239     def test_submit(self, stubs, tm, arvdock):
 240         capture_stdout = cStringIO.StringIO()
 241         exited = arvados_cwl.main(
 242             ["--submit", "--no-wait", "--api=jobs", "--debug",
 243              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 244             capture_stdout, sys.stderr, api_client=stubs.api)
 245         self.assertEqual(exited, 0)
 246
 247         stubs.api.collections().create.assert_has_calls([
 248             mock.call(),
 249             mock.call(body=JsonDiffMatcher({
 250                 'manifest_text':
 251                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 252                 'replication_desired': None,
 253                 'name': 'submit_tool.cwl dependencies',
 254             }), ensure_unique_name=True),
 255             mock.call().execute(num_retries=4),
 256             mock.call(body=JsonDiffMatcher({
 257                 'manifest_text':
 258                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 259                 'replication_desired': None,
 260                 'name': 'submit_wf.cwl input',
 261             }), ensure_unique_name=True),
 262             mock.call().execute(num_retries=4)])
 263
 264         arvdock.assert_has_calls([
 265             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 266             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 267         ])
 268
 269         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 270         stubs.api.pipeline_instances().create.assert_called_with(
 271             body=JsonDiffMatcher(expect_pipeline))
 272         self.assertEqual(capture_stdout.getvalue(),
 273                          stubs.expect_pipeline_uuid + '\n')
 274
 275
 276     @mock.patch("time.sleep")
 277     @stubs
 278     def test_submit_no_reuse(self, stubs, tm):
 279         capture_stdout = cStringIO.StringIO()
 280         exited = arvados_cwl.main(
 281             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 282              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 283             capture_stdout, sys.stderr, api_client=stubs.api)
 284         self.assertEqual(exited, 0)
 285
 286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 287         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 288
 289         stubs.api.pipeline_instances().create.assert_called_with(
 290             body=JsonDiffMatcher(expect_pipeline))
 291         self.assertEqual(capture_stdout.getvalue(),
 292                          stubs.expect_pipeline_uuid + '\n')
 293
 294     @mock.patch("time.sleep")
 295     @stubs
 296     def test_submit_on_error(self, stubs, tm):
 297         capture_stdout = cStringIO.StringIO()
 298         exited = arvados_cwl.main(
 299             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 301             capture_stdout, sys.stderr, api_client=stubs.api)
 302         self.assertEqual(exited, 0)
 303
 304         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 305         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 306
 307         stubs.api.pipeline_instances().create.assert_called_with(
 308             body=JsonDiffMatcher(expect_pipeline))
 309         self.assertEqual(capture_stdout.getvalue(),
 310                          stubs.expect_pipeline_uuid + '\n')
 311
 312
 313     @mock.patch("time.sleep")
 314     @stubs
 315     def test_submit_runner_ram(self, stubs, tm):
 316         capture_stdout = cStringIO.StringIO()
 317         exited = arvados_cwl.main(
 318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 0)
 322
 323         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 324         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331
 332     @mock.patch("time.sleep")
 333     @stubs
 334     def test_submit_invalid_runner_ram(self, stubs, tm):
 335         capture_stdout = cStringIO.StringIO()
 336         exited = arvados_cwl.main(
 337             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 338              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 339             capture_stdout, sys.stderr, api_client=stubs.api)
 340         self.assertEqual(exited, 1)
 341
 342     @mock.patch("time.sleep")
 343     @stubs
 344     def test_submit_output_name(self, stubs, tm):
 345         output_name = "test_output_name"
 346
 347         capture_stdout = cStringIO.StringIO()
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             capture_stdout, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 0)
 353
 354         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 355         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 356
 357         stubs.api.pipeline_instances().create.assert_called_with(
 358             body=JsonDiffMatcher(expect_pipeline))
 359         self.assertEqual(capture_stdout.getvalue(),
 360                          stubs.expect_pipeline_uuid + '\n')
 361
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_pipeline_name(self, stubs, tm):
 366         capture_stdout = cStringIO.StringIO()
 367         exited = arvados_cwl.main(
 368             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 369              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 370             capture_stdout, sys.stderr, api_client=stubs.api)
 371         self.assertEqual(exited, 0)
 372
 373         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 374         expect_pipeline["name"] = "hello job 123"
 375
 376         stubs.api.pipeline_instances().create.assert_called_with(
 377             body=JsonDiffMatcher(expect_pipeline))
 378         self.assertEqual(capture_stdout.getvalue(),
 379                          stubs.expect_pipeline_uuid + '\n')
 380
 381     @mock.patch("time.sleep")
 382     @stubs
 383     def test_submit_output_tags(self, stubs, tm):
 384         output_tags = "tag0,tag1,tag2"
 385
 386         capture_stdout = cStringIO.StringIO()
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 390             capture_stdout, sys.stderr, api_client=stubs.api)
 391         self.assertEqual(exited, 0)
 392
 393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 395
 396         stubs.api.pipeline_instances().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_pipeline))
 398         self.assertEqual(capture_stdout.getvalue(),
 399                          stubs.expect_pipeline_uuid + '\n')
 400
 401     @mock.patch("time.sleep")
 402     @stubs
 403     def test_submit_with_project_uuid(self, stubs, tm):
 404         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 405
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait",
 408              "--project-uuid", project_uuid,
 409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             sys.stdout, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 414         expect_pipeline["owner_uuid"] = project_uuid
 415         stubs.api.pipeline_instances().create.assert_called_with(
 416             body=JsonDiffMatcher(expect_pipeline))
 417
 418     @stubs
 419     def test_submit_container(self, stubs):
 420         capture_stdout = cStringIO.StringIO()
 421         try:
 422             exited = arvados_cwl.main(
 423                 ["--submit", "--no-wait", "--api=containers", "--debug",
 424                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 426             self.assertEqual(exited, 0)
 427         except:
 428             logging.exception("")
 429
 430         stubs.api.collections().create.assert_has_calls([
 431             mock.call(),
 432             mock.call(body=JsonDiffMatcher({
 433                 'manifest_text':
 434                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 435                 'replication_desired': None,
 436                 'name': 'submit_tool.cwl dependencies',
 437             }), ensure_unique_name=True),
 438             mock.call().execute(num_retries=4),
 439             mock.call(body=JsonDiffMatcher({
 440                 'manifest_text':
 441                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 442                 'replication_desired': None,
 443                 'name': 'submit_wf.cwl input',
 444             }), ensure_unique_name=True),
 445             mock.call().execute(num_retries=4)])
 446
 447         expect_container = copy.deepcopy(stubs.expect_container_spec)
 448         stubs.api.container_requests().create.assert_called_with(
 449             body=JsonDiffMatcher(expect_container))
 450         self.assertEqual(capture_stdout.getvalue(),
 451                          stubs.expect_container_request_uuid + '\n')
 452
 453     @stubs
 454     def test_submit_container_no_reuse(self, stubs):
 455         capture_stdout = cStringIO.StringIO()
 456         try:
 457             exited = arvados_cwl.main(
 458                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 459                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 460                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 461             self.assertEqual(exited, 0)
 462         except:
 463             logging.exception("")
 464
 465         expect_container = copy.deepcopy(stubs.expect_container_spec)
 466         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 467                                                   '--disable-reuse', '--on-error=continue',
 468                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 469
 470         stubs.api.container_requests().create.assert_called_with(
 471             body=JsonDiffMatcher(expect_container))
 472         self.assertEqual(capture_stdout.getvalue(),
 473                          stubs.expect_container_request_uuid + '\n')
 474
 475
 476     @stubs
 477     def test_submit_container_on_error(self, stubs):
 478         capture_stdout = cStringIO.StringIO()
 479         try:
 480             exited = arvados_cwl.main(
 481                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 482                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 483                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 484             self.assertEqual(exited, 0)
 485         except:
 486             logging.exception("")
 487
 488         expect_container = copy.deepcopy(stubs.expect_container_spec)
 489         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 490                                                   '--enable-reuse', '--on-error=stop',
 491                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 492
 493         stubs.api.container_requests().create.assert_called_with(
 494             body=JsonDiffMatcher(expect_container))
 495         self.assertEqual(capture_stdout.getvalue(),
 496                          stubs.expect_container_request_uuid + '\n')
 497
 498     @stubs
 499     def test_submit_container_output_name(self, stubs):
 500         output_name = "test_output_name"
 501
 502         capture_stdout = cStringIO.StringIO()
 503         try:
 504             exited = arvados_cwl.main(
 505                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 506                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 507                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 508             self.assertEqual(exited, 0)
 509         except:
 510             logging.exception("")
 511
 512         expect_container = copy.deepcopy(stubs.expect_container_spec)
 513         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 514                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 515                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 516         expect_container["output_name"] = output_name
 517
 518         stubs.api.container_requests().create.assert_called_with(
 519             body=JsonDiffMatcher(expect_container))
 520         self.assertEqual(capture_stdout.getvalue(),
 521                          stubs.expect_container_request_uuid + '\n')
 522
 523     @stubs
 524     def test_submit_container_output_tags(self, stubs):
 525         output_tags = "tag0,tag1,tag2"
 526
 527         capture_stdout = cStringIO.StringIO()
 528         try:
 529             exited = arvados_cwl.main(
 530                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 531                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533             self.assertEqual(exited, 0)
 534         except:
 535             logging.exception("")
 536
 537         expect_container = copy.deepcopy(stubs.expect_container_spec)
 538         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 539                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 540                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 541
 542         stubs.api.container_requests().create.assert_called_with(
 543             body=JsonDiffMatcher(expect_container))
 544         self.assertEqual(capture_stdout.getvalue(),
 545                          stubs.expect_container_request_uuid + '\n')
 546
 547     @stubs
 548     def test_submit_container_runner_ram(self, stubs):
 549         capture_stdout = cStringIO.StringIO()
 550         try:
 551             exited = arvados_cwl.main(
 552                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 553                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 554                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 555             self.assertEqual(exited, 0)
 556         except:
 557             logging.exception("")
 558
 559         expect_container = copy.deepcopy(stubs.expect_container_spec)
 560         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 561
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566
 567     @mock.patch("arvados.collection.CollectionReader")
 568     @mock.patch("time.sleep")
 569     @stubs
 570     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 571         capture_stdout = cStringIO.StringIO()
 572         exited = arvados_cwl.main(
 573             ["--submit", "--no-wait", "--api=containers", "--debug",
 574              "tests/wf/submit_keepref_wf.cwl"],
 575             capture_stdout, sys.stderr, api_client=stubs.api)
 576         self.assertEqual(exited, 0)
 577
 578
 579     @mock.patch("arvados.collection.CollectionReader")
 580     @mock.patch("time.sleep")
 581     @stubs
 582     def test_submit_keepref(self, stubs, tm, reader):
 583         capture_stdout = cStringIO.StringIO()
 584
 585         with open("tests/wf/expect_arvworkflow.cwl") as f:
 586             reader().open().__enter__().read.return_value = f.read()
 587
 588         exited = arvados_cwl.main(
 589             ["--submit", "--no-wait", "--api=containers", "--debug",
 590              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 591             capture_stdout, sys.stderr, api_client=stubs.api)
 592         self.assertEqual(exited, 0)
 593
 594         expect_container = {
 595             'priority': 1,
 596             'mounts': {
 597                 '/var/spool/cwl': {
 598                     'writable': True,
 599                     'kind': 'collection'
 600                 },
 601                 'stdout': {
 602                     'path': '/var/spool/cwl/cwl.output.json',
 603                     'kind': 'file'
 604                 },
 605                 '/var/lib/cwl/workflow': {
 606                     'portable_data_hash': '99999999999999999999999999999994+99',
 607                     'kind': 'collection'
 608                 },
 609                 '/var/lib/cwl/cwl.input.json': {
 610                     'content': {
 611                         'x': 'XxX'
 612                     },
 613                     'kind': 'json'
 614                 }
 615             }, 'state': 'Committed',
 616             'owner_uuid': None,
 617             'output_path': '/var/spool/cwl',
 618             'name': 'expect_arvworkflow.cwl#main',
 619             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 620             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 621                         '--enable-reuse', '--on-error=continue',
 622                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 623             'cwd': '/var/spool/cwl',
 624             'runtime_constraints': {
 625                 'API': True,
 626                 'vcpus': 1,
 627                 'ram': 1073741824
 628             },
 629             "properties": {}
 630         }
 631
 632         stubs.api.container_requests().create.assert_called_with(
 633             body=JsonDiffMatcher(expect_container))
 634         self.assertEqual(capture_stdout.getvalue(),
 635                          stubs.expect_container_request_uuid + '\n')
 636
 637
 638     @mock.patch("arvados.collection.CollectionReader")
 639     @mock.patch("time.sleep")
 640     @stubs
 641     def test_submit_jobs_keepref(self, stubs, tm, reader):
 642         capture_stdout = cStringIO.StringIO()
 643
 644         with open("tests/wf/expect_arvworkflow.cwl") as f:
 645             reader().open().__enter__().read.return_value = f.read()
 646
 647         exited = arvados_cwl.main(
 648             ["--submit", "--no-wait", "--api=jobs", "--debug",
 649              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 650             capture_stdout, sys.stderr, api_client=stubs.api)
 651         self.assertEqual(exited, 0)
 652
 653         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 654         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 655         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 656         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 657         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 658         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 659         stubs.api.pipeline_instances().create.assert_called_with(
 660             body=JsonDiffMatcher(expect_pipeline))
 661
 662     @mock.patch("time.sleep")
 663     @stubs
 664     def test_submit_arvworkflow(self, stubs, tm):
 665         capture_stdout = cStringIO.StringIO()
 666
 667         with open("tests/wf/expect_arvworkflow.cwl") as f:
 668             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 669
 670         exited = arvados_cwl.main(
 671             ["--submit", "--no-wait", "--api=containers", "--debug",
 672              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 673             capture_stdout, sys.stderr, api_client=stubs.api)
 674         self.assertEqual(exited, 0)
 675
 676         expect_container = {
 677             'priority': 1,
 678             'mounts': {
 679                 '/var/spool/cwl': {
 680                     'writable': True,
 681                     'kind': 'collection'
 682                 },
 683                 'stdout': {
 684                     'path': '/var/spool/cwl/cwl.output.json',
 685                     'kind': 'file'
 686                 },
 687                 '/var/lib/cwl/workflow.json': {
 688                     'kind': 'json',
 689                     'content': {
 690                         'cwlVersion': 'v1.0',
 691                         '$graph': [
 692                             {
 693                                 'id': '#main',
 694                                 'inputs': [
 695                                     {'type': 'string', 'id': '#main/x'}
 696                                 ],
 697                                 'steps': [
 698                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 699                                      'run': '#submit_tool.cwl',
 700                                      'id': '#main/step1',
 701                                      'out': []}
 702                                 ],
 703                                 'class': 'Workflow',
 704                                 'outputs': []
 705                             },
 706                             {
 707                                 'inputs': [
 708                                     {
 709                                         'inputBinding': {'position': 1},
 710                                         'type': 'string',
 711                                         'id': '#submit_tool.cwl/x'}
 712                                 ],
 713                                 'requirements': [
 714                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 715                                 ],
 716                                 'id': '#submit_tool.cwl',
 717                                 'outputs': [],
 718                                 'baseCommand': 'cat',
 719                                 'class': 'CommandLineTool'
 720                             }
 721                         ]
 722                     }
 723                 },
 724                 '/var/lib/cwl/cwl.input.json': {
 725                     'content': {
 726                         'x': 'XxX'
 727                     },
 728                     'kind': 'json'
 729                 }
 730             }, 'state': 'Committed',
 731             'owner_uuid': None,
 732             'output_path': '/var/spool/cwl',
 733             'name': 'a test workflow',
 734             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 735             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 736                         '--enable-reuse', '--on-error=continue',
 737                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 738             'cwd': '/var/spool/cwl',
 739             'runtime_constraints': {
 740                 'API': True,
 741                 'vcpus': 1,
 742                 'ram': 1073741824
 743             },
 744             "properties": {
 745                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 746             }
 747         }
 748
 749         stubs.api.container_requests().create.assert_called_with(
 750             body=JsonDiffMatcher(expect_container))
 751         self.assertEqual(capture_stdout.getvalue(),
 752                          stubs.expect_container_request_uuid + '\n')
 753
 754
 755     @stubs
 756     def test_submit_container_name(self, stubs):
 757         capture_stdout = cStringIO.StringIO()
 758         try:
 759             exited = arvados_cwl.main(
 760                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 761                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 762                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 763             self.assertEqual(exited, 0)
 764         except:
 765             logging.exception("")
 766
 767         expect_container = copy.deepcopy(stubs.expect_container_spec)
 768         expect_container["name"] = "hello container 123"
 769
 770         stubs.api.container_requests().create.assert_called_with(
 771             body=JsonDiffMatcher(expect_container))
 772         self.assertEqual(capture_stdout.getvalue(),
 773                          stubs.expect_container_request_uuid + '\n')
 774
 775
 776     @stubs
 777     def test_submit_job_runner_image(self, stubs):
 778         capture_stdout = cStringIO.StringIO()
 779         try:
 780             exited = arvados_cwl.main(
 781                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 782                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 783                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 784             self.assertEqual(exited, 0)
 785         except:
 786             logging.exception("")
 787
 788         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 789
 790         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 791         stubs.api.pipeline_instances().create.assert_called_with(
 792             body=JsonDiffMatcher(expect_pipeline))
 793         self.assertEqual(capture_stdout.getvalue(),
 794                          stubs.expect_pipeline_uuid + '\n')
 795
 796     @stubs
 797     def test_submit_container_runner_image(self, stubs):
 798         capture_stdout = cStringIO.StringIO()
 799         try:
 800             exited = arvados_cwl.main(
 801                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 802                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 803                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 804             self.assertEqual(exited, 0)
 805         except:
 806             logging.exception("")
 807
 808         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 809
 810         expect_container = copy.deepcopy(stubs.expect_container_spec)
 811         stubs.api.container_requests().create.assert_called_with(
 812             body=JsonDiffMatcher(expect_container))
 813         self.assertEqual(capture_stdout.getvalue(),
 814                          stubs.expect_container_request_uuid + '\n')
 815
 816
 817     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 818     @mock.patch("cwltool.docker.get_image")
 819     @mock.patch("arvados.api")
 820     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 821         arvrunner = mock.MagicMock()
 822         arvrunner.project_uuid = ""
 823         api.return_value = mock.MagicMock()
 824         arvrunner.api = api.return_value
 825         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 826                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 827                                                                         "link_class": "docker_image_repo+tag",
 828                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 829                                                                         "owner_uuid": "",
 830                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 831                                                             {"items": [{"created_at": "",
 832                                                                         "head_uuid": "",
 833                                                                         "link_class": "docker_image_hash",
 834                                                                         "name": "123456",
 835                                                                         "owner_uuid": "",
 836                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 837         )
 838         find_one_image_hash.return_value = "123456"
 839
 840         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 841                                                                               "owner_uuid": "",
 842                                                                               "manifest_text": "",
 843                                                                               "properties": ""
 844                                                                           }], "items_available": 1, "offset": 0},)
 845         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 846         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 847                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 848
 849 class TestCreateTemplate(unittest.TestCase):
 850     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 851
 852     def _adjust_script_params(self, expect_component):
 853         expect_component['script_parameters']['x'] = {
 854             'dataclass': 'File',
 855             'required': True,
 856             'type': 'File',
 857             'value': '99999999999999999999999999999992+99/blorp.txt',
 858         }
 859         expect_component['script_parameters']['y'] = {
 860             'dataclass': 'Collection',
 861             'required': True,
 862             'type': 'Directory',
 863             'value': '99999999999999999999999999999998+99',
 864         }
 865         expect_component['script_parameters']['z'] = {
 866             'dataclass': 'Collection',
 867             'required': True,
 868             'type': 'Directory',
 869         }
 870
 871     @stubs
 872     def test_create(self, stubs):
 873         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 874
 875         capture_stdout = cStringIO.StringIO()
 876
 877         exited = arvados_cwl.main(
 878             ["--create-workflow", "--debug",
 879              "--api=jobs",
 880              "--project-uuid", project_uuid,
 881              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 882             capture_stdout, sys.stderr, api_client=stubs.api)
 883         self.assertEqual(exited, 0)
 884
 885         stubs.api.pipeline_instances().create.refute_called()
 886         stubs.api.jobs().create.refute_called()
 887
 888         expect_component = copy.deepcopy(stubs.expect_job_spec)
 889         self._adjust_script_params(expect_component)
 890         expect_template = {
 891             "components": {
 892                 "submit_wf.cwl": expect_component,
 893             },
 894             "name": "submit_wf.cwl",
 895             "owner_uuid": project_uuid,
 896         }
 897         stubs.api.pipeline_templates().create.assert_called_with(
 898             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 899
 900         self.assertEqual(capture_stdout.getvalue(),
 901                          stubs.expect_pipeline_template_uuid + '\n')
 902
 903
 904     @stubs
 905     def test_create_name(self, stubs):
 906         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 907
 908         capture_stdout = cStringIO.StringIO()
 909
 910         exited = arvados_cwl.main(
 911             ["--create-workflow", "--debug",
 912              "--project-uuid", project_uuid,
 913              "--api=jobs",
 914              "--name", "testing 123",
 915              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 916             capture_stdout, sys.stderr, api_client=stubs.api)
 917         self.assertEqual(exited, 0)
 918
 919         stubs.api.pipeline_instances().create.refute_called()
 920         stubs.api.jobs().create.refute_called()
 921
 922         expect_component = copy.deepcopy(stubs.expect_job_spec)
 923         self._adjust_script_params(expect_component)
 924         expect_template = {
 925             "components": {
 926                 "testing 123": expect_component,
 927             },
 928             "name": "testing 123",
 929             "owner_uuid": project_uuid,
 930         }
 931         stubs.api.pipeline_templates().create.assert_called_with(
 932             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 933
 934         self.assertEqual(capture_stdout.getvalue(),
 935                          stubs.expect_pipeline_template_uuid + '\n')
 936
 937
 938     @stubs
 939     def test_update_name(self, stubs):
 940         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 941
 942         capture_stdout = cStringIO.StringIO()
 943
 944         exited = arvados_cwl.main(
 945             ["--update-workflow", self.existing_template_uuid,
 946              "--debug",
 947              "--project-uuid", project_uuid,
 948              "--api=jobs",
 949              "--name", "testing 123",
 950              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 951             capture_stdout, sys.stderr, api_client=stubs.api)
 952         self.assertEqual(exited, 0)
 953
 954         stubs.api.pipeline_instances().create.refute_called()
 955         stubs.api.jobs().create.refute_called()
 956
 957         expect_component = copy.deepcopy(stubs.expect_job_spec)
 958         self._adjust_script_params(expect_component)
 959         expect_template = {
 960             "components": {
 961                 "testing 123": expect_component,
 962             },
 963             "name": "testing 123",
 964             "owner_uuid": project_uuid,
 965         }
 966         stubs.api.pipeline_templates().create.refute_called()
 967         stubs.api.pipeline_templates().update.assert_called_with(
 968             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 969
 970         self.assertEqual(capture_stdout.getvalue(),
 971                          self.existing_template_uuid + '\n')
 972
 973
 974 class TestCreateWorkflow(unittest.TestCase):
 975     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 976     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 977
 978     @stubs
 979     def test_create(self, stubs):
 980         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 981
 982         capture_stdout = cStringIO.StringIO()
 983
 984         exited = arvados_cwl.main(
 985             ["--create-workflow", "--debug",
 986              "--api=containers",
 987              "--project-uuid", project_uuid,
 988              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 989             capture_stdout, sys.stderr, api_client=stubs.api)
 990         self.assertEqual(exited, 0)
 991
 992         stubs.api.pipeline_templates().create.refute_called()
 993         stubs.api.container_requests().create.refute_called()
 994
 995         body = {
 996             "workflow": {
 997                 "owner_uuid": project_uuid,
 998                 "name": "submit_wf.cwl",
 999                 "description": "",
1000                 "definition": self.expect_workflow,
1001             }
1002         }
1003         stubs.api.workflows().create.assert_called_with(
1004             body=JsonDiffMatcher(body))
1005
1006         self.assertEqual(capture_stdout.getvalue(),
1007                          stubs.expect_workflow_uuid + '\n')
1008
1009
1010     @stubs
1011     def test_create_name(self, stubs):
1012         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1013
1014         capture_stdout = cStringIO.StringIO()
1015
1016         exited = arvados_cwl.main(
1017             ["--create-workflow", "--debug",
1018              "--api=containers",
1019              "--project-uuid", project_uuid,
1020              "--name", "testing 123",
1021              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1022             capture_stdout, sys.stderr, api_client=stubs.api)
1023         self.assertEqual(exited, 0)
1024
1025         stubs.api.pipeline_templates().create.refute_called()
1026         stubs.api.container_requests().create.refute_called()
1027
1028         body = {
1029             "workflow": {
1030                 "owner_uuid": project_uuid,
1031                 "name": "testing 123",
1032                 "description": "",
1033                 "definition": self.expect_workflow,
1034             }
1035         }
1036         stubs.api.workflows().create.assert_called_with(
1037             body=JsonDiffMatcher(body))
1038
1039         self.assertEqual(capture_stdout.getvalue(),
1040                          stubs.expect_workflow_uuid + '\n')
1041
1042     @stubs
1043     def test_incompatible_api(self, stubs):
1044         capture_stderr = cStringIO.StringIO()
1045         logging.getLogger('arvados.cwl-runner').addHandler(
1046             logging.StreamHandler(capture_stderr))
1047
1048         exited = arvados_cwl.main(
1049             ["--update-workflow", self.existing_workflow_uuid,
1050              "--api=jobs",
1051              "--debug",
1052              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1053             sys.stderr, sys.stderr, api_client=stubs.api)
1054         self.assertEqual(exited, 1)
1055         self.assertRegexpMatches(
1056             capture_stderr.getvalue(),
1057             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1058
1059     @stubs
1060     def test_update(self, stubs):
1061         capture_stdout = cStringIO.StringIO()
1062
1063         exited = arvados_cwl.main(
1064             ["--update-workflow", self.existing_workflow_uuid,
1065              "--debug",
1066              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1067             capture_stdout, sys.stderr, api_client=stubs.api)
1068         self.assertEqual(exited, 0)
1069
1070         body = {
1071             "workflow": {
1072                 "name": "submit_wf.cwl",
1073                 "description": "",
1074                 "definition": self.expect_workflow,
1075             }
1076         }
1077         stubs.api.workflows().update.assert_called_with(
1078             uuid=self.existing_workflow_uuid,
1079             body=JsonDiffMatcher(body))
1080         self.assertEqual(capture_stdout.getvalue(),
1081                          self.existing_workflow_uuid + '\n')
1082
1083
1084     @stubs
1085     def test_update_name(self, stubs):
1086         capture_stdout = cStringIO.StringIO()
1087
1088         exited = arvados_cwl.main(
1089             ["--update-workflow", self.existing_workflow_uuid,
1090              "--debug", "--name", "testing 123",
1091              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1092             capture_stdout, sys.stderr, api_client=stubs.api)
1093         self.assertEqual(exited, 0)
1094
1095         body = {
1096             "workflow": {
1097                 "name": "testing 123",
1098                 "description": "",
1099                 "definition": self.expect_workflow,
1100             }
1101         }
1102         stubs.api.workflows().update.assert_called_with(
1103             uuid=self.existing_workflow_uuid,
1104             body=JsonDiffMatcher(body))
1105         self.assertEqual(capture_stdout.getvalue(),
1106                          self.existing_workflow_uuid + '\n')
1107
1108
1109 class TestTemplateInputs(unittest.TestCase):
1110     expect_template = {
1111         "components": {
1112             "inputs_test.cwl": {
1113                 'runtime_constraints': {
1114                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1115                     'min_ram_mb_per_node': 1024
1116                 },
1117                 'script_parameters': {
1118                     'cwl:tool':
1119                     '99999999999999999999999999999992+99/workflow.cwl#main',
1120                     'optionalFloatInput': None,
1121                     'fileInput': {
1122                         'type': 'File',
1123                         'dataclass': 'File',
1124                         'required': True,
1125                         'title': "It's a file; we expect to find some characters in it.",
1126                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1127                     },
1128                     'floatInput': {
1129                         'type': 'float',
1130                         'dataclass': 'number',
1131                         'required': True,
1132                         'title': 'Floats like a duck',
1133                         'default': 0.1,
1134                         'value': 0.1,
1135                     },
1136                     'optionalFloatInput': {
1137                         'type': ['null', 'float'],
1138                         'dataclass': 'number',
1139                         'required': False,
1140                     },
1141                     'boolInput': {
1142                         'type': 'boolean',
1143                         'dataclass': 'boolean',
1144                         'required': True,
1145                         'title': 'True or false?',
1146                     },
1147                 },
1148                 'repository': 'arvados',
1149                 'script_version': 'master',
1150                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1151                 'script': 'cwl-runner',
1152             },
1153         },
1154         "name": "inputs_test.cwl",
1155     }
1156
1157     @stubs
1158     def test_inputs_empty(self, stubs):
1159         exited = arvados_cwl.main(
1160             ["--create-template",
1161              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1162             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1163         self.assertEqual(exited, 0)
1164
1165         stubs.api.pipeline_templates().create.assert_called_with(
1166             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1167
1168     @stubs
1169     def test_inputs(self, stubs):
1170         exited = arvados_cwl.main(
1171             ["--create-template",
1172              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1173             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1174         self.assertEqual(exited, 0)
1175
1176         expect_template = copy.deepcopy(self.expect_template)
1177         params = expect_template[
1178             "components"]["inputs_test.cwl"]["script_parameters"]
1179         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
1180         params["cwl:tool"] = '99999999999999999999999999999994+99/workflow.cwl#main'
1181         params["floatInput"]["value"] = 1.234
1182         params["boolInput"]["value"] = True
1183
1184         stubs.api.pipeline_templates().create.assert_called_with(
1185             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)