sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 import ruamel.yaml as yaml
  21
  22 _rootDesc = None
  23
  24 def stubs(func):
  25     @functools.wraps(func)
  26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  27     @mock.patch("arvados.collection.KeepClient")
  28     @mock.patch("arvados.keep.KeepClient")
  29     @mock.patch("arvados.events.subscribe")
  30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  31         class Stubs:
  32             pass
  33         stubs = Stubs()
  34         stubs.events = events
  35         stubs.keepdocker = keepdocker
  36
  37
  38         def putstub(p, **kwargs):
  39             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  40         keep_client1().put.side_effect = putstub
  41         keep_client1.put.side_effect = putstub
  42         keep_client2().put.side_effect = putstub
  43         keep_client2.put.side_effect = putstub
  44
  45         stubs.keep_client = keep_client2
  46         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  47         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  48
  49         stubs.api = mock.MagicMock()
  50         stubs.api._rootDesc = get_rootDesc()
  51
  52         stubs.api.users().current().execute.return_value = {
  53             "uuid": stubs.fake_user_uuid,
  54         }
  55         stubs.api.collections().list().execute.return_value = {"items": []}
  56         stubs.api.collections().create().execute.side_effect = ({
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  58             "portable_data_hash": "99999999999999999999999999999991+99",
  59             "manifest_text": ""
  60         }, {
  61             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  62             "portable_data_hash": "99999999999999999999999999999992+99",
  63             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  64         },
  65         {
  66             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  67             "portable_data_hash": "99999999999999999999999999999994+99",
  68             "manifest_text": ""
  69         },
  70         {
  71             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  72             "portable_data_hash": "99999999999999999999999999999995+99",
  73             "manifest_text": ""
  74         },
  75         {
  76             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  77             "portable_data_hash": "99999999999999999999999999999996+99",
  78             "manifest_text": ""
  79         }
  80         )
  81         stubs.api.collections().get().execute.return_value = {
  82             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  83
  84         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  85         stubs.api.jobs().create().execute.return_value = {
  86             "uuid": stubs.expect_job_uuid,
  87             "state": "Queued",
  88         }
  89
  90         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  91         stubs.api.container_requests().create().execute.return_value = {
  92             "uuid": stubs.expect_container_request_uuid,
  93             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  94             "state": "Queued"
  95         }
  96
  97         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  98         stubs.api.pipeline_templates().create().execute.return_value = {
  99             "uuid": stubs.expect_pipeline_template_uuid,
 100         }
 101         stubs.expect_job_spec = {
 102             'runtime_constraints': {
 103                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 104                 'min_ram_mb_per_node': 1024
 105             },
 106             'script_parameters': {
 107                 'x': {
 108                     'basename': 'blorp.txt',
 109                     'location': 'keep:99999999999999999999999999999992+99/blorp.txt',
 110                     'class': 'File'
 111                 },
 112                 'y': {
 113                     'basename': '99999999999999999999999999999998+99',
 114                     'location': 'keep:99999999999999999999999999999998+99',
 115                     'class': 'Directory'
 116                 },
 117                 'z': {
 118                     'basename': 'anonymous',
 119                     "listing": [{
 120                         "basename": "renamed.txt",
 121                         "class": "File",
 122                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 123                     }],
 124                     'class': 'Directory'
 125                 },
 126                 'cwl:tool':
 127                 'f57578d5cfda7f70fef00cbc4b621e6b+58/workflow.cwl#main'
 128             },
 129             'repository': 'arvados',
 130             'script_version': 'master',
 131             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 132             'script': 'cwl-runner'
 133         }
 134         stubs.pipeline_component = stubs.expect_job_spec.copy()
 135         stubs.expect_pipeline_instance = {
 136             'name': 'submit_wf.cwl',
 137             'state': 'RunningOnServer',
 138             'owner_uuid': None,
 139             "components": {
 140                 "cwl-runner": {
 141                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 142                     'script_parameters': {
 143                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 144                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999992+99/blorp.txt'}},
 145                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 146                               'listing': [
 147                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 148                               ]}},
 149                         'cwl:tool': 'f57578d5cfda7f70fef00cbc4b621e6b+58/workflow.cwl#main',
 150                         'arv:enable_reuse': True,
 151                         'arv:on_error': 'continue'
 152                     },
 153                     'repository': 'arvados',
 154                     'script_version': 'master',
 155                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 156                     'script': 'cwl-runner',
 157                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 158                 }
 159             }
 160         }
 161         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 162         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 163         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 164         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 165         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 166             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 167             "state": "Queued"
 168         }
 169         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 170         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 171
 172         with open("tests/wf/submit_wf_packed.cwl") as f:
 173             expect_packed_workflow = yaml.round_trip_load(f)
 174
 175         stubs.expect_container_spec = {
 176             'priority': 1,
 177             'mounts': {
 178                 '/var/spool/cwl': {
 179                     'writable': True,
 180                     'kind': 'collection'
 181                 },
 182                 '/var/lib/cwl/workflow.json': {
 183                     'content': expect_packed_workflow,
 184                     'kind': 'json'
 185                 },
 186                 'stdout': {
 187                     'path': '/var/spool/cwl/cwl.output.json',
 188                     'kind': 'file'
 189                 },
 190                 '/var/lib/cwl/cwl.input.json': {
 191                     'kind': 'json',
 192                     'content': {
 193                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 194                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999992+99/blorp.txt'},
 195                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 196                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 197                         ]}
 198                     },
 199                     'kind': 'json'
 200                 }
 201             },
 202             'state': 'Committed',
 203             'owner_uuid': None,
 204             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 205                         '--enable-reuse', '--on-error=continue',
 206                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 207             'name': 'submit_wf.cwl',
 208             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 209             'output_path': '/var/spool/cwl',
 210             'cwd': '/var/spool/cwl',
 211             'runtime_constraints': {
 212                 'API': True,
 213                 'vcpus': 1,
 214                 'ram': 1024*1024*1024
 215             },
 216             "properties": {}
 217         }
 218
 219         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 220         stubs.api.workflows().create().execute.return_value = {
 221             "uuid": stubs.expect_workflow_uuid,
 222         }
 223         def update_mock(**kwargs):
 224             stubs.updated_uuid = kwargs.get('uuid')
 225             return mock.DEFAULT
 226         stubs.api.workflows().update.side_effect = update_mock
 227         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 228             "uuid": stubs.updated_uuid,
 229         }
 230
 231         return func(self, stubs, *args, **kwargs)
 232     return wrapped
 233
 234
 235 class TestSubmit(unittest.TestCase):
 236     @mock.patch("time.sleep")
 237     @stubs
 238     def test_submit(self, stubs, tm):
 239         capture_stdout = cStringIO.StringIO()
 240         exited = arvados_cwl.main(
 241             ["--submit", "--no-wait", "--api=jobs", "--debug",
 242              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 243             capture_stdout, sys.stderr, api_client=stubs.api)
 244         self.assertEqual(exited, 0)
 245
 246         stubs.api.collections().create.assert_has_calls([
 247             mock.call(),
 248             mock.call(body=JsonDiffMatcher({
 249                 'manifest_text':
 250                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 251                 'owner_uuid': None,
 252                 'name': 'submit_tool.cwl dependencies',
 253             }), ensure_unique_name=True),
 254             mock.call().execute(),
 255             mock.call(body=JsonDiffMatcher({
 256                 'manifest_text':
 257                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 258                 'owner_uuid': None,
 259                 'name': 'submit_wf.cwl input',
 260             }), ensure_unique_name=True),
 261             mock.call().execute()])
 262
 263         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 264         stubs.api.pipeline_instances().create.assert_called_with(
 265             body=JsonDiffMatcher(expect_pipeline))
 266         self.assertEqual(capture_stdout.getvalue(),
 267                          stubs.expect_pipeline_uuid + '\n')
 268
 269
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit_no_reuse(self, stubs, tm):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 281         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 282
 283         stubs.api.pipeline_instances().create.assert_called_with(
 284             body=JsonDiffMatcher(expect_pipeline))
 285         self.assertEqual(capture_stdout.getvalue(),
 286                          stubs.expect_pipeline_uuid + '\n')
 287
 288     @mock.patch("time.sleep")
 289     @stubs
 290     def test_submit_on_error(self, stubs, tm):
 291         capture_stdout = cStringIO.StringIO()
 292         exited = arvados_cwl.main(
 293             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 294              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 295             capture_stdout, sys.stderr, api_client=stubs.api)
 296         self.assertEqual(exited, 0)
 297
 298         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 299         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 300
 301         stubs.api.pipeline_instances().create.assert_called_with(
 302             body=JsonDiffMatcher(expect_pipeline))
 303         self.assertEqual(capture_stdout.getvalue(),
 304                          stubs.expect_pipeline_uuid + '\n')
 305
 306
 307     @mock.patch("time.sleep")
 308     @stubs
 309     def test_submit_runner_ram(self, stubs, tm):
 310         capture_stdout = cStringIO.StringIO()
 311         exited = arvados_cwl.main(
 312             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 313              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 314             capture_stdout, sys.stderr, api_client=stubs.api)
 315         self.assertEqual(exited, 0)
 316
 317         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 318         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 319
 320         stubs.api.pipeline_instances().create.assert_called_with(
 321             body=JsonDiffMatcher(expect_pipeline))
 322         self.assertEqual(capture_stdout.getvalue(),
 323                          stubs.expect_pipeline_uuid + '\n')
 324
 325
 326     @mock.patch("time.sleep")
 327     @stubs
 328     def test_submit_invalid_runner_ram(self, stubs, tm):
 329         capture_stdout = cStringIO.StringIO()
 330         exited = arvados_cwl.main(
 331             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 332              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 333             capture_stdout, sys.stderr, api_client=stubs.api)
 334         self.assertEqual(exited, 1)
 335
 336     @mock.patch("time.sleep")
 337     @stubs
 338     def test_submit_output_name(self, stubs, tm):
 339         output_name = "test_output_name"
 340
 341         capture_stdout = cStringIO.StringIO()
 342         exited = arvados_cwl.main(
 343             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 344              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 345             capture_stdout, sys.stderr, api_client=stubs.api)
 346         self.assertEqual(exited, 0)
 347
 348         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 349         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 350
 351         stubs.api.pipeline_instances().create.assert_called_with(
 352             body=JsonDiffMatcher(expect_pipeline))
 353         self.assertEqual(capture_stdout.getvalue(),
 354                          stubs.expect_pipeline_uuid + '\n')
 355
 356
 357     @mock.patch("time.sleep")
 358     @stubs
 359     def test_submit_pipeline_name(self, stubs, tm):
 360         capture_stdout = cStringIO.StringIO()
 361         exited = arvados_cwl.main(
 362             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 363              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 364             capture_stdout, sys.stderr, api_client=stubs.api)
 365         self.assertEqual(exited, 0)
 366
 367         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 368         expect_pipeline["name"] = "hello job 123"
 369
 370         stubs.api.pipeline_instances().create.assert_called_with(
 371             body=expect_pipeline)
 372         self.assertEqual(capture_stdout.getvalue(),
 373                          stubs.expect_pipeline_uuid + '\n')
 374
 375     @mock.patch("time.sleep")
 376     @stubs
 377     def test_submit_output_tags(self, stubs, tm):
 378         output_tags = "tag0,tag1,tag2"
 379
 380         capture_stdout = cStringIO.StringIO()
 381         exited = arvados_cwl.main(
 382             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 383              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 384             capture_stdout, sys.stderr, api_client=stubs.api)
 385         self.assertEqual(exited, 0)
 386
 387         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 388         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 389
 390         stubs.api.pipeline_instances().create.assert_called_with(
 391             body=JsonDiffMatcher(expect_pipeline))
 392         self.assertEqual(capture_stdout.getvalue(),
 393                          stubs.expect_pipeline_uuid + '\n')
 394
 395     @mock.patch("time.sleep")
 396     @stubs
 397     def test_submit_with_project_uuid(self, stubs, tm):
 398         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 399
 400         exited = arvados_cwl.main(
 401             ["--submit", "--no-wait",
 402              "--project-uuid", project_uuid,
 403              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 404             sys.stdout, sys.stderr, api_client=stubs.api)
 405         self.assertEqual(exited, 0)
 406
 407         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 408         expect_pipeline["owner_uuid"] = project_uuid
 409         stubs.api.pipeline_instances().create.assert_called_with(
 410             body=JsonDiffMatcher(expect_pipeline))
 411
 412     @stubs
 413     def test_submit_container(self, stubs):
 414         capture_stdout = cStringIO.StringIO()
 415         try:
 416             exited = arvados_cwl.main(
 417                 ["--submit", "--no-wait", "--api=containers", "--debug",
 418                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 419                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 420             self.assertEqual(exited, 0)
 421         except:
 422             logging.exception("")
 423
 424         stubs.api.collections().create.assert_has_calls([
 425             mock.call(),
 426             mock.call(body=JsonDiffMatcher({
 427                 'manifest_text':
 428                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 429                 'owner_uuid': None,
 430                 'name': 'submit_tool.cwl dependencies',
 431             }), ensure_unique_name=True),
 432             mock.call().execute(),
 433             mock.call(body=JsonDiffMatcher({
 434                 'manifest_text':
 435                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 436                 'owner_uuid': None,
 437                 'name': 'submit_wf.cwl input',
 438             }), ensure_unique_name=True),
 439             mock.call().execute()])
 440
 441         expect_container = copy.deepcopy(stubs.expect_container_spec)
 442         stubs.api.container_requests().create.assert_called_with(
 443             body=JsonDiffMatcher(expect_container))
 444         self.assertEqual(capture_stdout.getvalue(),
 445                          stubs.expect_container_request_uuid + '\n')
 446
 447     @stubs
 448     def test_submit_container_no_reuse(self, stubs):
 449         capture_stdout = cStringIO.StringIO()
 450         try:
 451             exited = arvados_cwl.main(
 452                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 453                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 454                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 455             self.assertEqual(exited, 0)
 456         except:
 457             logging.exception("")
 458
 459         expect_container = copy.deepcopy(stubs.expect_container_spec)
 460         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 461                                                   '--disable-reuse', '--on-error=continue',
 462                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 463
 464         stubs.api.container_requests().create.assert_called_with(
 465             body=JsonDiffMatcher(expect_container))
 466         self.assertEqual(capture_stdout.getvalue(),
 467                          stubs.expect_container_request_uuid + '\n')
 468
 469
 470     @stubs
 471     def test_submit_container_on_error(self, stubs):
 472         capture_stdout = cStringIO.StringIO()
 473         try:
 474             exited = arvados_cwl.main(
 475                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 476                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 477                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 478             self.assertEqual(exited, 0)
 479         except:
 480             logging.exception("")
 481
 482         expect_container = copy.deepcopy(stubs.expect_container_spec)
 483         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 484                                                   '--enable-reuse', '--on-error=stop',
 485                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 486
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_container))
 489         self.assertEqual(capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491
 492     @stubs
 493     def test_submit_container_output_name(self, stubs):
 494         output_name = "test_output_name"
 495
 496         capture_stdout = cStringIO.StringIO()
 497         try:
 498             exited = arvados_cwl.main(
 499                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 500                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 501                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 502             self.assertEqual(exited, 0)
 503         except:
 504             logging.exception("")
 505
 506         expect_container = copy.deepcopy(stubs.expect_container_spec)
 507         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 508                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 509                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 510         expect_container["output_name"] = output_name
 511
 512         stubs.api.container_requests().create.assert_called_with(
 513             body=JsonDiffMatcher(expect_container))
 514         self.assertEqual(capture_stdout.getvalue(),
 515                          stubs.expect_container_request_uuid + '\n')
 516
 517     @stubs
 518     def test_submit_container_output_tags(self, stubs):
 519         output_tags = "tag0,tag1,tag2"
 520
 521         capture_stdout = cStringIO.StringIO()
 522         try:
 523             exited = arvados_cwl.main(
 524                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 525                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 526                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 527             self.assertEqual(exited, 0)
 528         except:
 529             logging.exception("")
 530
 531         expect_container = copy.deepcopy(stubs.expect_container_spec)
 532         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 533                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 534                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 535
 536         stubs.api.container_requests().create.assert_called_with(
 537             body=JsonDiffMatcher(expect_container))
 538         self.assertEqual(capture_stdout.getvalue(),
 539                          stubs.expect_container_request_uuid + '\n')
 540
 541     @stubs
 542     def test_submit_container_runner_ram(self, stubs):
 543         capture_stdout = cStringIO.StringIO()
 544         try:
 545             exited = arvados_cwl.main(
 546                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 547                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 548                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 549             self.assertEqual(exited, 0)
 550         except:
 551             logging.exception("")
 552
 553         expect_container = copy.deepcopy(stubs.expect_container_spec)
 554         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 555
 556         stubs.api.container_requests().create.assert_called_with(
 557             body=JsonDiffMatcher(expect_container))
 558         self.assertEqual(capture_stdout.getvalue(),
 559                          stubs.expect_container_request_uuid + '\n')
 560
 561     @mock.patch("arvados.collection.CollectionReader")
 562     @mock.patch("time.sleep")
 563     @stubs
 564     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 565         capture_stdout = cStringIO.StringIO()
 566         exited = arvados_cwl.main(
 567             ["--submit", "--no-wait", "--api=containers", "--debug",
 568              "tests/wf/submit_keepref_wf.cwl"],
 569             capture_stdout, sys.stderr, api_client=stubs.api)
 570         self.assertEqual(exited, 0)
 571
 572
 573     @mock.patch("arvados.collection.CollectionReader")
 574     @mock.patch("time.sleep")
 575     @stubs
 576     def test_submit_keepref(self, stubs, tm, reader):
 577         capture_stdout = cStringIO.StringIO()
 578
 579         with open("tests/wf/expect_arvworkflow.cwl") as f:
 580             reader().open().__enter__().read.return_value = f.read()
 581
 582         exited = arvados_cwl.main(
 583             ["--submit", "--no-wait", "--api=containers", "--debug",
 584              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 585             capture_stdout, sys.stderr, api_client=stubs.api)
 586         self.assertEqual(exited, 0)
 587
 588         expect_container = {
 589             'priority': 1,
 590             'mounts': {
 591                 '/var/spool/cwl': {
 592                     'writable': True,
 593                     'kind': 'collection'
 594                 },
 595                 'stdout': {
 596                     'path': '/var/spool/cwl/cwl.output.json',
 597                     'kind': 'file'
 598                 },
 599                 '/var/lib/cwl/workflow': {
 600                     'portable_data_hash': '99999999999999999999999999999994+99',
 601                     'kind': 'collection'
 602                 },
 603                 '/var/lib/cwl/cwl.input.json': {
 604                     'content': {
 605                         'x': 'XxX'
 606                     },
 607                     'kind': 'json'
 608                 }
 609             }, 'state': 'Committed',
 610             'owner_uuid': None,
 611             'output_path': '/var/spool/cwl',
 612             'name': 'expect_arvworkflow.cwl#main',
 613             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 614             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 615                         '--enable-reuse', '--on-error=continue',
 616                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 617             'cwd': '/var/spool/cwl',
 618             'runtime_constraints': {
 619                 'API': True,
 620                 'vcpus': 1,
 621                 'ram': 1073741824
 622             },
 623             "properties": {}
 624         }
 625
 626         stubs.api.container_requests().create.assert_called_with(
 627             body=JsonDiffMatcher(expect_container))
 628         self.assertEqual(capture_stdout.getvalue(),
 629                          stubs.expect_container_request_uuid + '\n')
 630
 631
 632     @mock.patch("arvados.collection.CollectionReader")
 633     @mock.patch("time.sleep")
 634     @stubs
 635     def test_submit_jobs_keepref(self, stubs, tm, reader):
 636         capture_stdout = cStringIO.StringIO()
 637
 638         with open("tests/wf/expect_arvworkflow.cwl") as f:
 639             reader().open().__enter__().read.return_value = f.read()
 640
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=jobs", "--debug",
 643              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 644             capture_stdout, sys.stderr, api_client=stubs.api)
 645         self.assertEqual(exited, 0)
 646
 647         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 648         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 649         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 650         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 651         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 652         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 653         stubs.api.pipeline_instances().create.assert_called_with(
 654             body=JsonDiffMatcher(expect_pipeline))
 655
 656     @mock.patch("time.sleep")
 657     @stubs
 658     def test_submit_arvworkflow(self, stubs, tm):
 659         capture_stdout = cStringIO.StringIO()
 660
 661         with open("tests/wf/expect_arvworkflow.cwl") as f:
 662             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 663
 664         exited = arvados_cwl.main(
 665             ["--submit", "--no-wait", "--api=containers", "--debug",
 666              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 667             capture_stdout, sys.stderr, api_client=stubs.api)
 668         self.assertEqual(exited, 0)
 669
 670         expect_container = {
 671             'priority': 1,
 672             'mounts': {
 673                 '/var/spool/cwl': {
 674                     'writable': True,
 675                     'kind': 'collection'
 676                 },
 677                 'stdout': {
 678                     'path': '/var/spool/cwl/cwl.output.json',
 679                     'kind': 'file'
 680                 },
 681                 '/var/lib/cwl/workflow.json': {
 682                     'kind': 'json',
 683                     'content': {
 684                         'cwlVersion': 'v1.0',
 685                         '$graph': [
 686                             {
 687                                 'id': '#main',
 688                                 'inputs': [
 689                                     {'type': 'string', 'id': '#main/x'}
 690                                 ],
 691                                 'steps': [
 692                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 693                                      'run': '#submit_tool.cwl',
 694                                      'id': '#main/step1',
 695                                      'out': []}
 696                                 ],
 697                                 'class': 'Workflow',
 698                                 'outputs': []
 699                             },
 700                             {
 701                                 'inputs': [
 702                                     {
 703                                         'inputBinding': {'position': 1},
 704                                         'type': 'string',
 705                                         'id': '#submit_tool.cwl/x'}
 706                                 ],
 707                                 'requirements': [
 708                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 709                                 ],
 710                                 'id': '#submit_tool.cwl',
 711                                 'outputs': [],
 712                                 'baseCommand': 'cat',
 713                                 'class': 'CommandLineTool'
 714                             }
 715                         ]
 716                     }
 717                 },
 718                 '/var/lib/cwl/cwl.input.json': {
 719                     'content': {
 720                         'x': 'XxX'
 721                     },
 722                     'kind': 'json'
 723                 }
 724             }, 'state': 'Committed',
 725             'owner_uuid': None,
 726             'output_path': '/var/spool/cwl',
 727             'name': 'a test workflow',
 728             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 729             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 730                         '--enable-reuse', '--on-error=continue',
 731                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 732             'cwd': '/var/spool/cwl',
 733             'runtime_constraints': {
 734                 'API': True,
 735                 'vcpus': 1,
 736                 'ram': 1073741824
 737             },
 738             "properties": {
 739                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 740             }
 741         }
 742
 743         stubs.api.container_requests().create.assert_called_with(
 744             body=JsonDiffMatcher(expect_container))
 745         self.assertEqual(capture_stdout.getvalue(),
 746                          stubs.expect_container_request_uuid + '\n')
 747
 748
 749     @stubs
 750     def test_submit_container_name(self, stubs):
 751         capture_stdout = cStringIO.StringIO()
 752         try:
 753             exited = arvados_cwl.main(
 754                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 755                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 756                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 757             self.assertEqual(exited, 0)
 758         except:
 759             logging.exception("")
 760
 761         expect_container = copy.deepcopy(stubs.expect_container_spec)
 762         expect_container["name"] = "hello container 123"
 763
 764         stubs.api.container_requests().create.assert_called_with(
 765             body=JsonDiffMatcher(expect_container))
 766         self.assertEqual(capture_stdout.getvalue(),
 767                          stubs.expect_container_request_uuid + '\n')
 768
 769
 770     @stubs
 771     def test_submit_job_runner_image(self, stubs):
 772         capture_stdout = cStringIO.StringIO()
 773         try:
 774             exited = arvados_cwl.main(
 775                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 776                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 777                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 778             self.assertEqual(exited, 0)
 779         except:
 780             logging.exception("")
 781
 782         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 783
 784         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 785         stubs.api.pipeline_instances().create.assert_called_with(
 786             body=JsonDiffMatcher(expect_pipeline))
 787         self.assertEqual(capture_stdout.getvalue(),
 788                          stubs.expect_pipeline_uuid + '\n')
 789
 790     @stubs
 791     def test_submit_container_runner_image(self, stubs):
 792         capture_stdout = cStringIO.StringIO()
 793         try:
 794             exited = arvados_cwl.main(
 795                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 796                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 797                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 798             self.assertEqual(exited, 0)
 799         except:
 800             logging.exception("")
 801
 802         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 803
 804         expect_container = copy.deepcopy(stubs.expect_container_spec)
 805         stubs.api.container_requests().create.assert_called_with(
 806             body=JsonDiffMatcher(expect_container))
 807         self.assertEqual(capture_stdout.getvalue(),
 808                          stubs.expect_container_request_uuid + '\n')
 809
 810
 811     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 812     @mock.patch("cwltool.docker.get_image")
 813     @mock.patch("arvados.api")
 814     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 815         arvrunner = mock.MagicMock()
 816         arvrunner.project_uuid = ""
 817         api.return_value = mock.MagicMock()
 818         arvrunner.api = api.return_value
 819         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 820                                                             {"items": [], "items_available": 0, "offset": 0},
 821                                                             {"items": [], "items_available": 0, "offset": 0},
 822                                                             {"items": [{"created_at": "",
 823                                                                         "head_uuid": "",
 824                                                                         "link_class": "docker_image_hash",
 825                                                                         "name": "123456",
 826                                                                         "owner_uuid": "",
 827                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 828                                                             {"items": [], "items_available": 0, "offset": 0},
 829                                                             {"items": [{"created_at": "",
 830                                                                         "head_uuid": "",
 831                                                                         "link_class": "docker_image_repo+tag",
 832                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 833                                                                         "owner_uuid": "",
 834                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 835                                                             {"items": [{"created_at": "",
 836                                                                         "head_uuid": "",
 837                                                                         "link_class": "docker_image_hash",
 838                                                                         "name": "123456",
 839                                                                         "owner_uuid": "",
 840                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 841         )
 842         find_one_image_hash.return_value = "123456"
 843
 844         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 845                                                                   {"items": [{"uuid": "",
 846                                                                               "owner_uuid": "",
 847                                                                               "manifest_text": "",
 848                                                                               "properties": ""
 849                                                                           }], "items_available": 1, "offset": 0},
 850                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 851         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 852         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 853
 854 class TestCreateTemplate(unittest.TestCase):
 855     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 856
 857     def _adjust_script_params(self, expect_component):
 858         expect_component['script_parameters']['x'] = {
 859             'dataclass': 'File',
 860             'required': True,
 861             'type': 'File',
 862             'value': '99999999999999999999999999999992+99/blorp.txt',
 863         }
 864         expect_component['script_parameters']['y'] = {
 865             'dataclass': 'Collection',
 866             'required': True,
 867             'type': 'Directory',
 868             'value': '99999999999999999999999999999998+99',
 869         }
 870         expect_component['script_parameters']['z'] = {
 871             'dataclass': 'Collection',
 872             'required': True,
 873             'type': 'Directory',
 874         }
 875
 876     @stubs
 877     def test_create(self, stubs):
 878         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 879
 880         capture_stdout = cStringIO.StringIO()
 881
 882         exited = arvados_cwl.main(
 883             ["--create-workflow", "--debug",
 884              "--api=jobs",
 885              "--project-uuid", project_uuid,
 886              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 887             capture_stdout, sys.stderr, api_client=stubs.api)
 888         self.assertEqual(exited, 0)
 889
 890         stubs.api.pipeline_instances().create.refute_called()
 891         stubs.api.jobs().create.refute_called()
 892
 893         expect_component = copy.deepcopy(stubs.expect_job_spec)
 894         self._adjust_script_params(expect_component)
 895         expect_template = {
 896             "components": {
 897                 "submit_wf.cwl": expect_component,
 898             },
 899             "name": "submit_wf.cwl",
 900             "owner_uuid": project_uuid,
 901         }
 902         stubs.api.pipeline_templates().create.assert_called_with(
 903             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 904
 905         self.assertEqual(capture_stdout.getvalue(),
 906                          stubs.expect_pipeline_template_uuid + '\n')
 907
 908
 909     @stubs
 910     def test_create_name(self, stubs):
 911         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 912
 913         capture_stdout = cStringIO.StringIO()
 914
 915         exited = arvados_cwl.main(
 916             ["--create-workflow", "--debug",
 917              "--project-uuid", project_uuid,
 918              "--api=jobs",
 919              "--name", "testing 123",
 920              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 921             capture_stdout, sys.stderr, api_client=stubs.api)
 922         self.assertEqual(exited, 0)
 923
 924         stubs.api.pipeline_instances().create.refute_called()
 925         stubs.api.jobs().create.refute_called()
 926
 927         expect_component = copy.deepcopy(stubs.expect_job_spec)
 928         self._adjust_script_params(expect_component)
 929         expect_template = {
 930             "components": {
 931                 "testing 123": expect_component,
 932             },
 933             "name": "testing 123",
 934             "owner_uuid": project_uuid,
 935         }
 936         stubs.api.pipeline_templates().create.assert_called_with(
 937             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 938
 939         self.assertEqual(capture_stdout.getvalue(),
 940                          stubs.expect_pipeline_template_uuid + '\n')
 941
 942
 943     @stubs
 944     def test_update_name(self, stubs):
 945         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 946
 947         capture_stdout = cStringIO.StringIO()
 948
 949         exited = arvados_cwl.main(
 950             ["--update-workflow", self.existing_template_uuid,
 951              "--debug",
 952              "--project-uuid", project_uuid,
 953              "--api=jobs",
 954              "--name", "testing 123",
 955              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 956             capture_stdout, sys.stderr, api_client=stubs.api)
 957         self.assertEqual(exited, 0)
 958
 959         stubs.api.pipeline_instances().create.refute_called()
 960         stubs.api.jobs().create.refute_called()
 961
 962         expect_component = copy.deepcopy(stubs.expect_job_spec)
 963         self._adjust_script_params(expect_component)
 964         expect_template = {
 965             "components": {
 966                 "testing 123": expect_component,
 967             },
 968             "name": "testing 123",
 969             "owner_uuid": project_uuid,
 970         }
 971         stubs.api.pipeline_templates().create.refute_called()
 972         stubs.api.pipeline_templates().update.assert_called_with(
 973             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 974
 975         self.assertEqual(capture_stdout.getvalue(),
 976                          self.existing_template_uuid + '\n')
 977
 978
 979 class TestCreateWorkflow(unittest.TestCase):
 980     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 981     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 982
 983     @stubs
 984     def test_create(self, stubs):
 985         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 986
 987         capture_stdout = cStringIO.StringIO()
 988
 989         exited = arvados_cwl.main(
 990             ["--create-workflow", "--debug",
 991              "--api=containers",
 992              "--project-uuid", project_uuid,
 993              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994             capture_stdout, sys.stderr, api_client=stubs.api)
 995         self.assertEqual(exited, 0)
 996
 997         stubs.api.pipeline_templates().create.refute_called()
 998         stubs.api.container_requests().create.refute_called()
 999
1000         body = {
1001             "workflow": {
1002                 "owner_uuid": project_uuid,
1003                 "name": "submit_wf.cwl",
1004                 "description": "",
1005                 "definition": self.expect_workflow,
1006             }
1007         }
1008         stubs.api.workflows().create.assert_called_with(
1009             body=JsonDiffMatcher(body))
1010
1011         self.assertEqual(capture_stdout.getvalue(),
1012                          stubs.expect_workflow_uuid + '\n')
1013
1014
1015     @stubs
1016     def test_create_name(self, stubs):
1017         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1018
1019         capture_stdout = cStringIO.StringIO()
1020
1021         exited = arvados_cwl.main(
1022             ["--create-workflow", "--debug",
1023              "--api=containers",
1024              "--project-uuid", project_uuid,
1025              "--name", "testing 123",
1026              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1027             capture_stdout, sys.stderr, api_client=stubs.api)
1028         self.assertEqual(exited, 0)
1029
1030         stubs.api.pipeline_templates().create.refute_called()
1031         stubs.api.container_requests().create.refute_called()
1032
1033         body = {
1034             "workflow": {
1035                 "owner_uuid": project_uuid,
1036                 "name": "testing 123",
1037                 "description": "",
1038                 "definition": self.expect_workflow,
1039             }
1040         }
1041         stubs.api.workflows().create.assert_called_with(
1042             body=JsonDiffMatcher(body))
1043
1044         self.assertEqual(capture_stdout.getvalue(),
1045                          stubs.expect_workflow_uuid + '\n')
1046
1047     @stubs
1048     def test_incompatible_api(self, stubs):
1049         capture_stderr = cStringIO.StringIO()
1050         logging.getLogger('arvados.cwl-runner').addHandler(
1051             logging.StreamHandler(capture_stderr))
1052
1053         exited = arvados_cwl.main(
1054             ["--update-workflow", self.existing_workflow_uuid,
1055              "--api=jobs",
1056              "--debug",
1057              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1058             sys.stderr, sys.stderr, api_client=stubs.api)
1059         self.assertEqual(exited, 1)
1060         self.assertRegexpMatches(
1061             capture_stderr.getvalue(),
1062             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1063
1064     @stubs
1065     def test_update(self, stubs):
1066         capture_stdout = cStringIO.StringIO()
1067
1068         exited = arvados_cwl.main(
1069             ["--update-workflow", self.existing_workflow_uuid,
1070              "--debug",
1071              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1072             capture_stdout, sys.stderr, api_client=stubs.api)
1073         self.assertEqual(exited, 0)
1074
1075         body = {
1076             "workflow": {
1077                 "name": "submit_wf.cwl",
1078                 "description": "",
1079                 "definition": self.expect_workflow,
1080             }
1081         }
1082         stubs.api.workflows().update.assert_called_with(
1083             uuid=self.existing_workflow_uuid,
1084             body=JsonDiffMatcher(body))
1085         self.assertEqual(capture_stdout.getvalue(),
1086                          self.existing_workflow_uuid + '\n')
1087
1088
1089     @stubs
1090     def test_update_name(self, stubs):
1091         capture_stdout = cStringIO.StringIO()
1092
1093         exited = arvados_cwl.main(
1094             ["--update-workflow", self.existing_workflow_uuid,
1095              "--debug", "--name", "testing 123",
1096              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1097             capture_stdout, sys.stderr, api_client=stubs.api)
1098         self.assertEqual(exited, 0)
1099
1100         body = {
1101             "workflow": {
1102                 "name": "testing 123",
1103                 "description": "",
1104                 "definition": self.expect_workflow,
1105             }
1106         }
1107         stubs.api.workflows().update.assert_called_with(
1108             uuid=self.existing_workflow_uuid,
1109             body=JsonDiffMatcher(body))
1110         self.assertEqual(capture_stdout.getvalue(),
1111                          self.existing_workflow_uuid + '\n')
1112
1113
1114 class TestTemplateInputs(unittest.TestCase):
1115     expect_template = {
1116         "components": {
1117             "inputs_test.cwl": {
1118                 'runtime_constraints': {
1119                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1120                     'min_ram_mb_per_node': 1024
1121                 },
1122                 'script_parameters': {
1123                     'cwl:tool':
1124                     '5800682d508698dc9ce6d2fc618f21d8+58/workflow.cwl#main',
1125                     'optionalFloatInput': None,
1126                     'fileInput': {
1127                         'type': 'File',
1128                         'dataclass': 'File',
1129                         'required': True,
1130                         'title': "It's a file; we expect to find some characters in it.",
1131                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1132                     },
1133                     'floatInput': {
1134                         'type': 'float',
1135                         'dataclass': 'number',
1136                         'required': True,
1137                         'title': 'Floats like a duck',
1138                         'default': 0.1,
1139                         'value': 0.1,
1140                     },
1141                     'optionalFloatInput': {
1142                         'type': ['null', 'float'],
1143                         'dataclass': 'number',
1144                         'required': False,
1145                     },
1146                     'boolInput': {
1147                         'type': 'boolean',
1148                         'dataclass': 'boolean',
1149                         'required': True,
1150                         'title': 'True or false?',
1151                     },
1152                 },
1153                 'repository': 'arvados',
1154                 'script_version': 'master',
1155                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1156                 'script': 'cwl-runner',
1157             },
1158         },
1159         "name": "inputs_test.cwl",
1160     }
1161
1162     @stubs
1163     def test_inputs_empty(self, stubs):
1164         exited = arvados_cwl.main(
1165             ["--create-template",
1166              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1167             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1168         self.assertEqual(exited, 0)
1169
1170         stubs.api.pipeline_templates().create.assert_called_with(
1171             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1172
1173     @stubs
1174     def test_inputs(self, stubs):
1175         exited = arvados_cwl.main(
1176             ["--create-template",
1177              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1178             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1179         self.assertEqual(exited, 0)
1180
1181         expect_template = copy.deepcopy(self.expect_template)
1182         params = expect_template[
1183             "components"]["inputs_test.cwl"]["script_parameters"]
1184         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
1185         params["floatInput"]["value"] = 1.234
1186         params["boolInput"]["value"] = True
1187
1188         stubs.api.pipeline_templates().create.assert_called_with(
1189             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)