sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 import ruamel.yaml as yaml
  21
  22 _rootDesc = None
  23
  24 def stubs(func):
  25     @functools.wraps(func)
  26     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  27     @mock.patch("arvados.collection.KeepClient")
  28     @mock.patch("arvados.keep.KeepClient")
  29     @mock.patch("arvados.events.subscribe")
  30     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  31         class Stubs:
  32             pass
  33         stubs = Stubs()
  34         stubs.events = events
  35         stubs.keepdocker = keepdocker
  36
  37
  38         def putstub(p, **kwargs):
  39             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  40         keep_client1().put.side_effect = putstub
  41         keep_client1.put.side_effect = putstub
  42         keep_client2().put.side_effect = putstub
  43         keep_client2.put.side_effect = putstub
  44
  45         stubs.keep_client = keep_client2
  46         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  47         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  48
  49         stubs.api = mock.MagicMock()
  50         stubs.api._rootDesc = get_rootDesc()
  51
  52         stubs.api.users().current().execute.return_value = {
  53             "uuid": stubs.fake_user_uuid,
  54         }
  55         stubs.api.collections().list().execute.return_value = {"items": []}
  56         stubs.api.collections().create().execute.side_effect = ({
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  58             "portable_data_hash": "99999999999999999999999999999991+99",
  59             "manifest_text": ""
  60         }, {
  61             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  62             "portable_data_hash": "99999999999999999999999999999992+99",
  63             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  64         },
  65         {
  66             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  67             "portable_data_hash": "99999999999999999999999999999994+99",
  68             "manifest_text": ""
  69         },
  70         {
  71             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  72             "portable_data_hash": "99999999999999999999999999999995+99",
  73             "manifest_text": ""
  74         },
  75         {
  76             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  77             "portable_data_hash": "99999999999999999999999999999996+99",
  78             "manifest_text": ""
  79         }
  80         )
  81         stubs.api.collections().get().execute.return_value = {
  82             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  83
  84         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  85         stubs.api.jobs().create().execute.return_value = {
  86             "uuid": stubs.expect_job_uuid,
  87             "state": "Queued",
  88         }
  89
  90         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  91         stubs.api.container_requests().create().execute.return_value = {
  92             "uuid": stubs.expect_container_request_uuid,
  93             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  94             "state": "Queued"
  95         }
  96
  97         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  98         stubs.api.pipeline_templates().create().execute.return_value = {
  99             "uuid": stubs.expect_pipeline_template_uuid,
 100         }
 101         stubs.expect_job_spec = {
 102             'runtime_constraints': {
 103                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 104                 'min_ram_mb_per_node': 1024
 105             },
 106             'script_parameters': {
 107                 'x': {
 108                     'basename': 'blorp.txt',
 109                     'location': 'keep:99999999999999999999999999999992+99/blorp.txt',
 110                     'class': 'File'
 111                 },
 112                 'y': {
 113                     'basename': '99999999999999999999999999999998+99',
 114                     'location': 'keep:99999999999999999999999999999998+99',
 115                     'class': 'Directory'
 116                 },
 117                 'z': {
 118                     'basename': 'anonymous',
 119                     "listing": [{
 120                         "basename": "renamed.txt",
 121                         "class": "File",
 122                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 123                     }],
 124                     'class': 'Directory'
 125                 },
 126                 'cwl:tool':
 127                 '99999999999999999999999999999994+99/workflow.cwl#main'
 128             },
 129             'repository': 'arvados',
 130             'script_version': 'master',
 131             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 132             'script': 'cwl-runner'
 133         }
 134         stubs.pipeline_component = stubs.expect_job_spec.copy()
 135         stubs.expect_pipeline_instance = {
 136             'name': 'submit_wf.cwl',
 137             'state': 'RunningOnServer',
 138             'owner_uuid': None,
 139             "components": {
 140                 "cwl-runner": {
 141                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 142                     'script_parameters': {
 143                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 144                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999992+99/blorp.txt'}},
 145                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 146                               'listing': [
 147                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 148                               ]}},
 149                         'cwl:tool': '99999999999999999999999999999994+99/workflow.cwl#main',
 150                         'arv:enable_reuse': True,
 151                         'arv:on_error': 'continue'
 152                     },
 153                     'repository': 'arvados',
 154                     'script_version': 'master',
 155                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 156                     'script': 'cwl-runner',
 157                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 158                 }
 159             }
 160         }
 161         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 162         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 163         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 164         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 165         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 166             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 167             "state": "Queued"
 168         }
 169         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 170         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 171
 172         with open("tests/wf/submit_wf_packed.cwl") as f:
 173             expect_packed_workflow = yaml.round_trip_load(f)
 174
 175         stubs.expect_container_spec = {
 176             'priority': 1,
 177             'mounts': {
 178                 '/var/spool/cwl': {
 179                     'writable': True,
 180                     'kind': 'collection'
 181                 },
 182                 '/var/lib/cwl/workflow.json': {
 183                     'content': expect_packed_workflow,
 184                     'kind': 'json'
 185                 },
 186                 'stdout': {
 187                     'path': '/var/spool/cwl/cwl.output.json',
 188                     'kind': 'file'
 189                 },
 190                 '/var/lib/cwl/cwl.input.json': {
 191                     'kind': 'json',
 192                     'content': {
 193                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 194                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999992+99/blorp.txt'},
 195                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 196                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 197                         ]}
 198                     },
 199                     'kind': 'json'
 200                 }
 201             },
 202             'state': 'Committed',
 203             'owner_uuid': None,
 204             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 205                         '--enable-reuse', '--on-error=continue',
 206                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 207             'name': 'submit_wf.cwl',
 208             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 209             'output_path': '/var/spool/cwl',
 210             'cwd': '/var/spool/cwl',
 211             'runtime_constraints': {
 212                 'API': True,
 213                 'vcpus': 1,
 214                 'ram': 1024*1024*1024
 215             },
 216             "properties": {}
 217         }
 218
 219         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 220         stubs.api.workflows().create().execute.return_value = {
 221             "uuid": stubs.expect_workflow_uuid,
 222         }
 223         def update_mock(**kwargs):
 224             stubs.updated_uuid = kwargs.get('uuid')
 225             return mock.DEFAULT
 226         stubs.api.workflows().update.side_effect = update_mock
 227         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 228             "uuid": stubs.updated_uuid,
 229         }
 230
 231         return func(self, stubs, *args, **kwargs)
 232     return wrapped
 233
 234
 235 class TestSubmit(unittest.TestCase):
 236     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 237     @mock.patch("time.sleep")
 238     @stubs
 239     def test_submit(self, stubs, tm, arvdock):
 240         capture_stdout = cStringIO.StringIO()
 241         exited = arvados_cwl.main(
 242             ["--submit", "--no-wait", "--api=jobs", "--debug",
 243              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 244             capture_stdout, sys.stderr, api_client=stubs.api)
 245         self.assertEqual(exited, 0)
 246
 247         stubs.api.collections().create.assert_has_calls([
 248             mock.call(),
 249             mock.call(body=JsonDiffMatcher({
 250                 'manifest_text':
 251                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 252                 'replication_desired': None,
 253                 'name': 'submit_tool.cwl dependencies',
 254             }), ensure_unique_name=True),
 255             mock.call().execute(num_retries=4),
 256             mock.call(body=JsonDiffMatcher({
 257                 'manifest_text':
 258                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 259                 'replication_desired': None,
 260                 'name': 'submit_wf.cwl input',
 261             }), ensure_unique_name=True),
 262             mock.call().execute(num_retries=4)])
 263
 264         arvdock.assert_has_calls([
 265             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 266             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 267         ])
 268
 269         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 270         stubs.api.pipeline_instances().create.assert_called_with(
 271             body=JsonDiffMatcher(expect_pipeline))
 272         self.assertEqual(capture_stdout.getvalue(),
 273                          stubs.expect_pipeline_uuid + '\n')
 274
 275
 276     @mock.patch("time.sleep")
 277     @stubs
 278     def test_submit_no_reuse(self, stubs, tm):
 279         capture_stdout = cStringIO.StringIO()
 280         exited = arvados_cwl.main(
 281             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 282              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 283             capture_stdout, sys.stderr, api_client=stubs.api)
 284         self.assertEqual(exited, 0)
 285
 286         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 287         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 288
 289         stubs.api.pipeline_instances().create.assert_called_with(
 290             body=JsonDiffMatcher(expect_pipeline))
 291         self.assertEqual(capture_stdout.getvalue(),
 292                          stubs.expect_pipeline_uuid + '\n')
 293
 294     @mock.patch("time.sleep")
 295     @stubs
 296     def test_submit_on_error(self, stubs, tm):
 297         capture_stdout = cStringIO.StringIO()
 298         exited = arvados_cwl.main(
 299             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 301             capture_stdout, sys.stderr, api_client=stubs.api)
 302         self.assertEqual(exited, 0)
 303
 304         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 305         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 306
 307         stubs.api.pipeline_instances().create.assert_called_with(
 308             body=JsonDiffMatcher(expect_pipeline))
 309         self.assertEqual(capture_stdout.getvalue(),
 310                          stubs.expect_pipeline_uuid + '\n')
 311
 312
 313     @mock.patch("time.sleep")
 314     @stubs
 315     def test_submit_runner_ram(self, stubs, tm):
 316         capture_stdout = cStringIO.StringIO()
 317         exited = arvados_cwl.main(
 318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 0)
 322
 323         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 324         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331
 332     @mock.patch("time.sleep")
 333     @stubs
 334     def test_submit_invalid_runner_ram(self, stubs, tm):
 335         capture_stdout = cStringIO.StringIO()
 336         exited = arvados_cwl.main(
 337             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 338              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 339             capture_stdout, sys.stderr, api_client=stubs.api)
 340         self.assertEqual(exited, 1)
 341
 342     @mock.patch("time.sleep")
 343     @stubs
 344     def test_submit_output_name(self, stubs, tm):
 345         output_name = "test_output_name"
 346
 347         capture_stdout = cStringIO.StringIO()
 348         exited = arvados_cwl.main(
 349             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 351             capture_stdout, sys.stderr, api_client=stubs.api)
 352         self.assertEqual(exited, 0)
 353
 354         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 355         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 356
 357         stubs.api.pipeline_instances().create.assert_called_with(
 358             body=JsonDiffMatcher(expect_pipeline))
 359         self.assertEqual(capture_stdout.getvalue(),
 360                          stubs.expect_pipeline_uuid + '\n')
 361
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_pipeline_name(self, stubs, tm):
 366         capture_stdout = cStringIO.StringIO()
 367         exited = arvados_cwl.main(
 368             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 369              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 370             capture_stdout, sys.stderr, api_client=stubs.api)
 371         self.assertEqual(exited, 0)
 372
 373         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 374         expect_pipeline["name"] = "hello job 123"
 375
 376         stubs.api.pipeline_instances().create.assert_called_with(
 377             body=JsonDiffMatcher(expect_pipeline))
 378         self.assertEqual(capture_stdout.getvalue(),
 379                          stubs.expect_pipeline_uuid + '\n')
 380
 381     @mock.patch("time.sleep")
 382     @stubs
 383     def test_submit_output_tags(self, stubs, tm):
 384         output_tags = "tag0,tag1,tag2"
 385
 386         capture_stdout = cStringIO.StringIO()
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 390             capture_stdout, sys.stderr, api_client=stubs.api)
 391         self.assertEqual(exited, 0)
 392
 393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 395
 396         stubs.api.pipeline_instances().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_pipeline))
 398         self.assertEqual(capture_stdout.getvalue(),
 399                          stubs.expect_pipeline_uuid + '\n')
 400
 401     @mock.patch("time.sleep")
 402     @stubs
 403     def test_submit_with_project_uuid(self, stubs, tm):
 404         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 405
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait",
 408              "--project-uuid", project_uuid,
 409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             sys.stdout, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 414         expect_pipeline["owner_uuid"] = project_uuid
 415         stubs.api.pipeline_instances().create.assert_called_with(
 416             body=JsonDiffMatcher(expect_pipeline))
 417
 418     @stubs
 419     def test_submit_container(self, stubs):
 420         capture_stdout = cStringIO.StringIO()
 421         try:
 422             exited = arvados_cwl.main(
 423                 ["--submit", "--no-wait", "--api=containers", "--debug",
 424                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 425                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 426             self.assertEqual(exited, 0)
 427         except:
 428             logging.exception("")
 429
 430         stubs.api.collections().create.assert_has_calls([
 431             mock.call(),
 432             mock.call(body=JsonDiffMatcher({
 433                 'manifest_text':
 434                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 435                 'replication_desired': None,
 436                 'name': 'submit_tool.cwl dependencies',
 437             }), ensure_unique_name=True),
 438             mock.call().execute(num_retries=4),
 439             mock.call(body=JsonDiffMatcher({
 440                 'manifest_text':
 441                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 442                 'replication_desired': None,
 443                 'name': 'submit_wf.cwl input',
 444             }), ensure_unique_name=True),
 445             mock.call().execute(num_retries=4)])
 446
 447         expect_container = copy.deepcopy(stubs.expect_container_spec)
 448         stubs.api.container_requests().create.assert_called_with(
 449             body=JsonDiffMatcher(expect_container))
 450         self.assertEqual(capture_stdout.getvalue(),
 451                          stubs.expect_container_request_uuid + '\n')
 452
 453     @stubs
 454     def test_submit_container_no_reuse(self, stubs):
 455         capture_stdout = cStringIO.StringIO()
 456         try:
 457             exited = arvados_cwl.main(
 458                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 459                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 460                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 461             self.assertEqual(exited, 0)
 462         except:
 463             logging.exception("")
 464
 465         expect_container = copy.deepcopy(stubs.expect_container_spec)
 466         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 467                                                   '--disable-reuse', '--on-error=continue',
 468                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 469
 470         stubs.api.container_requests().create.assert_called_with(
 471             body=JsonDiffMatcher(expect_container))
 472         self.assertEqual(capture_stdout.getvalue(),
 473                          stubs.expect_container_request_uuid + '\n')
 474
 475
 476     @stubs
 477     def test_submit_container_on_error(self, stubs):
 478         capture_stdout = cStringIO.StringIO()
 479         try:
 480             exited = arvados_cwl.main(
 481                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 482                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 483                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 484             self.assertEqual(exited, 0)
 485         except:
 486             logging.exception("")
 487
 488         expect_container = copy.deepcopy(stubs.expect_container_spec)
 489         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 490                                                   '--enable-reuse', '--on-error=stop',
 491                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 492
 493         stubs.api.container_requests().create.assert_called_with(
 494             body=JsonDiffMatcher(expect_container))
 495         self.assertEqual(capture_stdout.getvalue(),
 496                          stubs.expect_container_request_uuid + '\n')
 497
 498     @stubs
 499     def test_submit_container_output_name(self, stubs):
 500         output_name = "test_output_name"
 501
 502         capture_stdout = cStringIO.StringIO()
 503         try:
 504             exited = arvados_cwl.main(
 505                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 506                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 507                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 508             self.assertEqual(exited, 0)
 509         except:
 510             logging.exception("")
 511
 512         expect_container = copy.deepcopy(stubs.expect_container_spec)
 513         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 514                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 515                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 516         expect_container["output_name"] = output_name
 517
 518         stubs.api.container_requests().create.assert_called_with(
 519             body=JsonDiffMatcher(expect_container))
 520         self.assertEqual(capture_stdout.getvalue(),
 521                          stubs.expect_container_request_uuid + '\n')
 522
 523
 524     @stubs
 525     def test_submit_container_output_ttl(self, stubs):
 526         capture_stdout = cStringIO.StringIO()
 527         try:
 528             exited = arvados_cwl.main(
 529                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 530                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 531                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 532             self.assertEqual(exited, 0)
 533         except:
 534             logging.exception("")
 535
 536         expect_container = copy.deepcopy(stubs.expect_container_spec)
 537         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 538                                        '--enable-reuse', '--on-error=continue',
 539                                        "--intermediate-output-ttl=3600",
 540                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 541
 542         stubs.api.container_requests().create.assert_called_with(
 543             body=JsonDiffMatcher(expect_container))
 544         self.assertEqual(capture_stdout.getvalue(),
 545                          stubs.expect_container_request_uuid + '\n')
 546
 547     @stubs
 548     def test_submit_container_output_tags(self, stubs):
 549         output_tags = "tag0,tag1,tag2"
 550
 551         capture_stdout = cStringIO.StringIO()
 552         try:
 553             exited = arvados_cwl.main(
 554                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 555                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 556                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 557             self.assertEqual(exited, 0)
 558         except:
 559             logging.exception("")
 560
 561         expect_container = copy.deepcopy(stubs.expect_container_spec)
 562         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 563                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 564                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 565
 566         stubs.api.container_requests().create.assert_called_with(
 567             body=JsonDiffMatcher(expect_container))
 568         self.assertEqual(capture_stdout.getvalue(),
 569                          stubs.expect_container_request_uuid + '\n')
 570
 571     @stubs
 572     def test_submit_container_runner_ram(self, stubs):
 573         capture_stdout = cStringIO.StringIO()
 574         try:
 575             exited = arvados_cwl.main(
 576                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 577                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 578                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 579             self.assertEqual(exited, 0)
 580         except:
 581             logging.exception("")
 582
 583         expect_container = copy.deepcopy(stubs.expect_container_spec)
 584         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 585
 586         stubs.api.container_requests().create.assert_called_with(
 587             body=JsonDiffMatcher(expect_container))
 588         self.assertEqual(capture_stdout.getvalue(),
 589                          stubs.expect_container_request_uuid + '\n')
 590
 591     @mock.patch("arvados.collection.CollectionReader")
 592     @mock.patch("time.sleep")
 593     @stubs
 594     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 595         capture_stdout = cStringIO.StringIO()
 596         exited = arvados_cwl.main(
 597             ["--submit", "--no-wait", "--api=containers", "--debug",
 598              "tests/wf/submit_keepref_wf.cwl"],
 599             capture_stdout, sys.stderr, api_client=stubs.api)
 600         self.assertEqual(exited, 0)
 601
 602
 603     @mock.patch("arvados.collection.CollectionReader")
 604     @mock.patch("time.sleep")
 605     @stubs
 606     def test_submit_keepref(self, stubs, tm, reader):
 607         capture_stdout = cStringIO.StringIO()
 608
 609         with open("tests/wf/expect_arvworkflow.cwl") as f:
 610             reader().open().__enter__().read.return_value = f.read()
 611
 612         exited = arvados_cwl.main(
 613             ["--submit", "--no-wait", "--api=containers", "--debug",
 614              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 615             capture_stdout, sys.stderr, api_client=stubs.api)
 616         self.assertEqual(exited, 0)
 617
 618         expect_container = {
 619             'priority': 1,
 620             'mounts': {
 621                 '/var/spool/cwl': {
 622                     'writable': True,
 623                     'kind': 'collection'
 624                 },
 625                 'stdout': {
 626                     'path': '/var/spool/cwl/cwl.output.json',
 627                     'kind': 'file'
 628                 },
 629                 '/var/lib/cwl/workflow': {
 630                     'portable_data_hash': '99999999999999999999999999999994+99',
 631                     'kind': 'collection'
 632                 },
 633                 '/var/lib/cwl/cwl.input.json': {
 634                     'content': {
 635                         'x': 'XxX'
 636                     },
 637                     'kind': 'json'
 638                 }
 639             }, 'state': 'Committed',
 640             'owner_uuid': None,
 641             'output_path': '/var/spool/cwl',
 642             'name': 'expect_arvworkflow.cwl#main',
 643             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 644             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 645                         '--enable-reuse', '--on-error=continue',
 646                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 647             'cwd': '/var/spool/cwl',
 648             'runtime_constraints': {
 649                 'API': True,
 650                 'vcpus': 1,
 651                 'ram': 1073741824
 652             },
 653             "properties": {}
 654         }
 655
 656         stubs.api.container_requests().create.assert_called_with(
 657             body=JsonDiffMatcher(expect_container))
 658         self.assertEqual(capture_stdout.getvalue(),
 659                          stubs.expect_container_request_uuid + '\n')
 660
 661
 662     @mock.patch("arvados.collection.CollectionReader")
 663     @mock.patch("time.sleep")
 664     @stubs
 665     def test_submit_jobs_keepref(self, stubs, tm, reader):
 666         capture_stdout = cStringIO.StringIO()
 667
 668         with open("tests/wf/expect_arvworkflow.cwl") as f:
 669             reader().open().__enter__().read.return_value = f.read()
 670
 671         exited = arvados_cwl.main(
 672             ["--submit", "--no-wait", "--api=jobs", "--debug",
 673              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 674             capture_stdout, sys.stderr, api_client=stubs.api)
 675         self.assertEqual(exited, 0)
 676
 677         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 678         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 679         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 680         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 681         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 682         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 683         stubs.api.pipeline_instances().create.assert_called_with(
 684             body=JsonDiffMatcher(expect_pipeline))
 685
 686     @mock.patch("time.sleep")
 687     @stubs
 688     def test_submit_arvworkflow(self, stubs, tm):
 689         capture_stdout = cStringIO.StringIO()
 690
 691         with open("tests/wf/expect_arvworkflow.cwl") as f:
 692             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 693
 694         exited = arvados_cwl.main(
 695             ["--submit", "--no-wait", "--api=containers", "--debug",
 696              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 697             capture_stdout, sys.stderr, api_client=stubs.api)
 698         self.assertEqual(exited, 0)
 699
 700         expect_container = {
 701             'priority': 1,
 702             'mounts': {
 703                 '/var/spool/cwl': {
 704                     'writable': True,
 705                     'kind': 'collection'
 706                 },
 707                 'stdout': {
 708                     'path': '/var/spool/cwl/cwl.output.json',
 709                     'kind': 'file'
 710                 },
 711                 '/var/lib/cwl/workflow.json': {
 712                     'kind': 'json',
 713                     'content': {
 714                         'cwlVersion': 'v1.0',
 715                         '$graph': [
 716                             {
 717                                 'id': '#main',
 718                                 'inputs': [
 719                                     {'type': 'string', 'id': '#main/x'}
 720                                 ],
 721                                 'steps': [
 722                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 723                                      'run': '#submit_tool.cwl',
 724                                      'id': '#main/step1',
 725                                      'out': []}
 726                                 ],
 727                                 'class': 'Workflow',
 728                                 'outputs': []
 729                             },
 730                             {
 731                                 'inputs': [
 732                                     {
 733                                         'inputBinding': {'position': 1},
 734                                         'type': 'string',
 735                                         'id': '#submit_tool.cwl/x'}
 736                                 ],
 737                                 'requirements': [
 738                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 739                                 ],
 740                                 'id': '#submit_tool.cwl',
 741                                 'outputs': [],
 742                                 'baseCommand': 'cat',
 743                                 'class': 'CommandLineTool'
 744                             }
 745                         ]
 746                     }
 747                 },
 748                 '/var/lib/cwl/cwl.input.json': {
 749                     'content': {
 750                         'x': 'XxX'
 751                     },
 752                     'kind': 'json'
 753                 }
 754             }, 'state': 'Committed',
 755             'owner_uuid': None,
 756             'output_path': '/var/spool/cwl',
 757             'name': 'a test workflow',
 758             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 759             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 760                         '--enable-reuse', '--on-error=continue',
 761                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 762             'cwd': '/var/spool/cwl',
 763             'runtime_constraints': {
 764                 'API': True,
 765                 'vcpus': 1,
 766                 'ram': 1073741824
 767             },
 768             "properties": {
 769                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 770             }
 771         }
 772
 773         stubs.api.container_requests().create.assert_called_with(
 774             body=JsonDiffMatcher(expect_container))
 775         self.assertEqual(capture_stdout.getvalue(),
 776                          stubs.expect_container_request_uuid + '\n')
 777
 778
 779     @stubs
 780     def test_submit_container_name(self, stubs):
 781         capture_stdout = cStringIO.StringIO()
 782         try:
 783             exited = arvados_cwl.main(
 784                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 785                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 786                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 787             self.assertEqual(exited, 0)
 788         except:
 789             logging.exception("")
 790
 791         expect_container = copy.deepcopy(stubs.expect_container_spec)
 792         expect_container["name"] = "hello container 123"
 793
 794         stubs.api.container_requests().create.assert_called_with(
 795             body=JsonDiffMatcher(expect_container))
 796         self.assertEqual(capture_stdout.getvalue(),
 797                          stubs.expect_container_request_uuid + '\n')
 798
 799
 800     @stubs
 801     def test_submit_job_runner_image(self, stubs):
 802         capture_stdout = cStringIO.StringIO()
 803         try:
 804             exited = arvados_cwl.main(
 805                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 806                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 807                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 808             self.assertEqual(exited, 0)
 809         except:
 810             logging.exception("")
 811
 812         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 813
 814         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 815         stubs.api.pipeline_instances().create.assert_called_with(
 816             body=JsonDiffMatcher(expect_pipeline))
 817         self.assertEqual(capture_stdout.getvalue(),
 818                          stubs.expect_pipeline_uuid + '\n')
 819
 820     @stubs
 821     def test_submit_container_runner_image(self, stubs):
 822         capture_stdout = cStringIO.StringIO()
 823         try:
 824             exited = arvados_cwl.main(
 825                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 826                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 827                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 828             self.assertEqual(exited, 0)
 829         except:
 830             logging.exception("")
 831
 832         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 833
 834         expect_container = copy.deepcopy(stubs.expect_container_spec)
 835         stubs.api.container_requests().create.assert_called_with(
 836             body=JsonDiffMatcher(expect_container))
 837         self.assertEqual(capture_stdout.getvalue(),
 838                          stubs.expect_container_request_uuid + '\n')
 839
 840
 841     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 842     @mock.patch("cwltool.docker.get_image")
 843     @mock.patch("arvados.api")
 844     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 845         arvrunner = mock.MagicMock()
 846         arvrunner.project_uuid = ""
 847         api.return_value = mock.MagicMock()
 848         arvrunner.api = api.return_value
 849         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 850                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 851                                                                         "link_class": "docker_image_repo+tag",
 852                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 853                                                                         "owner_uuid": "",
 854                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 855                                                             {"items": [{"created_at": "",
 856                                                                         "head_uuid": "",
 857                                                                         "link_class": "docker_image_hash",
 858                                                                         "name": "123456",
 859                                                                         "owner_uuid": "",
 860                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 861         )
 862         find_one_image_hash.return_value = "123456"
 863
 864         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 865                                                                               "owner_uuid": "",
 866                                                                               "manifest_text": "",
 867                                                                               "properties": ""
 868                                                                           }], "items_available": 1, "offset": 0},)
 869         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 870         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 871                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 872
 873 class TestCreateTemplate(unittest.TestCase):
 874     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 875
 876     def _adjust_script_params(self, expect_component):
 877         expect_component['script_parameters']['x'] = {
 878             'dataclass': 'File',
 879             'required': True,
 880             'type': 'File',
 881             'value': '99999999999999999999999999999992+99/blorp.txt',
 882         }
 883         expect_component['script_parameters']['y'] = {
 884             'dataclass': 'Collection',
 885             'required': True,
 886             'type': 'Directory',
 887             'value': '99999999999999999999999999999998+99',
 888         }
 889         expect_component['script_parameters']['z'] = {
 890             'dataclass': 'Collection',
 891             'required': True,
 892             'type': 'Directory',
 893         }
 894
 895     @stubs
 896     def test_create(self, stubs):
 897         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 898
 899         capture_stdout = cStringIO.StringIO()
 900
 901         exited = arvados_cwl.main(
 902             ["--create-workflow", "--debug",
 903              "--api=jobs",
 904              "--project-uuid", project_uuid,
 905              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 906             capture_stdout, sys.stderr, api_client=stubs.api)
 907         self.assertEqual(exited, 0)
 908
 909         stubs.api.pipeline_instances().create.refute_called()
 910         stubs.api.jobs().create.refute_called()
 911
 912         expect_component = copy.deepcopy(stubs.expect_job_spec)
 913         self._adjust_script_params(expect_component)
 914         expect_template = {
 915             "components": {
 916                 "submit_wf.cwl": expect_component,
 917             },
 918             "name": "submit_wf.cwl",
 919             "owner_uuid": project_uuid,
 920         }
 921         stubs.api.pipeline_templates().create.assert_called_with(
 922             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 923
 924         self.assertEqual(capture_stdout.getvalue(),
 925                          stubs.expect_pipeline_template_uuid + '\n')
 926
 927
 928     @stubs
 929     def test_create_name(self, stubs):
 930         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 931
 932         capture_stdout = cStringIO.StringIO()
 933
 934         exited = arvados_cwl.main(
 935             ["--create-workflow", "--debug",
 936              "--project-uuid", project_uuid,
 937              "--api=jobs",
 938              "--name", "testing 123",
 939              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 940             capture_stdout, sys.stderr, api_client=stubs.api)
 941         self.assertEqual(exited, 0)
 942
 943         stubs.api.pipeline_instances().create.refute_called()
 944         stubs.api.jobs().create.refute_called()
 945
 946         expect_component = copy.deepcopy(stubs.expect_job_spec)
 947         self._adjust_script_params(expect_component)
 948         expect_template = {
 949             "components": {
 950                 "testing 123": expect_component,
 951             },
 952             "name": "testing 123",
 953             "owner_uuid": project_uuid,
 954         }
 955         stubs.api.pipeline_templates().create.assert_called_with(
 956             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 957
 958         self.assertEqual(capture_stdout.getvalue(),
 959                          stubs.expect_pipeline_template_uuid + '\n')
 960
 961
 962     @stubs
 963     def test_update_name(self, stubs):
 964         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 965
 966         capture_stdout = cStringIO.StringIO()
 967
 968         exited = arvados_cwl.main(
 969             ["--update-workflow", self.existing_template_uuid,
 970              "--debug",
 971              "--project-uuid", project_uuid,
 972              "--api=jobs",
 973              "--name", "testing 123",
 974              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 975             capture_stdout, sys.stderr, api_client=stubs.api)
 976         self.assertEqual(exited, 0)
 977
 978         stubs.api.pipeline_instances().create.refute_called()
 979         stubs.api.jobs().create.refute_called()
 980
 981         expect_component = copy.deepcopy(stubs.expect_job_spec)
 982         self._adjust_script_params(expect_component)
 983         expect_template = {
 984             "components": {
 985                 "testing 123": expect_component,
 986             },
 987             "name": "testing 123",
 988             "owner_uuid": project_uuid,
 989         }
 990         stubs.api.pipeline_templates().create.refute_called()
 991         stubs.api.pipeline_templates().update.assert_called_with(
 992             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 993
 994         self.assertEqual(capture_stdout.getvalue(),
 995                          self.existing_template_uuid + '\n')
 996
 997
 998 class TestCreateWorkflow(unittest.TestCase):
 999     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1000     expect_workflow = open("tests/wf/expect_packed.cwl").read()
1001
1002     @stubs
1003     def test_create(self, stubs):
1004         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1005
1006         capture_stdout = cStringIO.StringIO()
1007
1008         exited = arvados_cwl.main(
1009             ["--create-workflow", "--debug",
1010              "--api=containers",
1011              "--project-uuid", project_uuid,
1012              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1013             capture_stdout, sys.stderr, api_client=stubs.api)
1014         self.assertEqual(exited, 0)
1015
1016         stubs.api.pipeline_templates().create.refute_called()
1017         stubs.api.container_requests().create.refute_called()
1018
1019         body = {
1020             "workflow": {
1021                 "owner_uuid": project_uuid,
1022                 "name": "submit_wf.cwl",
1023                 "description": "",
1024                 "definition": self.expect_workflow,
1025             }
1026         }
1027         stubs.api.workflows().create.assert_called_with(
1028             body=JsonDiffMatcher(body))
1029
1030         self.assertEqual(capture_stdout.getvalue(),
1031                          stubs.expect_workflow_uuid + '\n')
1032
1033
1034     @stubs
1035     def test_create_name(self, stubs):
1036         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1037
1038         capture_stdout = cStringIO.StringIO()
1039
1040         exited = arvados_cwl.main(
1041             ["--create-workflow", "--debug",
1042              "--api=containers",
1043              "--project-uuid", project_uuid,
1044              "--name", "testing 123",
1045              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1046             capture_stdout, sys.stderr, api_client=stubs.api)
1047         self.assertEqual(exited, 0)
1048
1049         stubs.api.pipeline_templates().create.refute_called()
1050         stubs.api.container_requests().create.refute_called()
1051
1052         body = {
1053             "workflow": {
1054                 "owner_uuid": project_uuid,
1055                 "name": "testing 123",
1056                 "description": "",
1057                 "definition": self.expect_workflow,
1058             }
1059         }
1060         stubs.api.workflows().create.assert_called_with(
1061             body=JsonDiffMatcher(body))
1062
1063         self.assertEqual(capture_stdout.getvalue(),
1064                          stubs.expect_workflow_uuid + '\n')
1065
1066     @stubs
1067     def test_incompatible_api(self, stubs):
1068         capture_stderr = cStringIO.StringIO()
1069         logging.getLogger('arvados.cwl-runner').addHandler(
1070             logging.StreamHandler(capture_stderr))
1071
1072         exited = arvados_cwl.main(
1073             ["--update-workflow", self.existing_workflow_uuid,
1074              "--api=jobs",
1075              "--debug",
1076              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1077             sys.stderr, sys.stderr, api_client=stubs.api)
1078         self.assertEqual(exited, 1)
1079         self.assertRegexpMatches(
1080             capture_stderr.getvalue(),
1081             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1082
1083     @stubs
1084     def test_update(self, stubs):
1085         capture_stdout = cStringIO.StringIO()
1086
1087         exited = arvados_cwl.main(
1088             ["--update-workflow", self.existing_workflow_uuid,
1089              "--debug",
1090              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1091             capture_stdout, sys.stderr, api_client=stubs.api)
1092         self.assertEqual(exited, 0)
1093
1094         body = {
1095             "workflow": {
1096                 "name": "submit_wf.cwl",
1097                 "description": "",
1098                 "definition": self.expect_workflow,
1099             }
1100         }
1101         stubs.api.workflows().update.assert_called_with(
1102             uuid=self.existing_workflow_uuid,
1103             body=JsonDiffMatcher(body))
1104         self.assertEqual(capture_stdout.getvalue(),
1105                          self.existing_workflow_uuid + '\n')
1106
1107
1108     @stubs
1109     def test_update_name(self, stubs):
1110         capture_stdout = cStringIO.StringIO()
1111
1112         exited = arvados_cwl.main(
1113             ["--update-workflow", self.existing_workflow_uuid,
1114              "--debug", "--name", "testing 123",
1115              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1116             capture_stdout, sys.stderr, api_client=stubs.api)
1117         self.assertEqual(exited, 0)
1118
1119         body = {
1120             "workflow": {
1121                 "name": "testing 123",
1122                 "description": "",
1123                 "definition": self.expect_workflow,
1124             }
1125         }
1126         stubs.api.workflows().update.assert_called_with(
1127             uuid=self.existing_workflow_uuid,
1128             body=JsonDiffMatcher(body))
1129         self.assertEqual(capture_stdout.getvalue(),
1130                          self.existing_workflow_uuid + '\n')
1131
1132
1133 class TestTemplateInputs(unittest.TestCase):
1134     expect_template = {
1135         "components": {
1136             "inputs_test.cwl": {
1137                 'runtime_constraints': {
1138                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1139                     'min_ram_mb_per_node': 1024
1140                 },
1141                 'script_parameters': {
1142                     'cwl:tool':
1143                     '99999999999999999999999999999992+99/workflow.cwl#main',
1144                     'optionalFloatInput': None,
1145                     'fileInput': {
1146                         'type': 'File',
1147                         'dataclass': 'File',
1148                         'required': True,
1149                         'title': "It's a file; we expect to find some characters in it.",
1150                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1151                     },
1152                     'floatInput': {
1153                         'type': 'float',
1154                         'dataclass': 'number',
1155                         'required': True,
1156                         'title': 'Floats like a duck',
1157                         'default': 0.1,
1158                         'value': 0.1,
1159                     },
1160                     'optionalFloatInput': {
1161                         'type': ['null', 'float'],
1162                         'dataclass': 'number',
1163                         'required': False,
1164                     },
1165                     'boolInput': {
1166                         'type': 'boolean',
1167                         'dataclass': 'boolean',
1168                         'required': True,
1169                         'title': 'True or false?',
1170                     },
1171                 },
1172                 'repository': 'arvados',
1173                 'script_version': 'master',
1174                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1175                 'script': 'cwl-runner',
1176             },
1177         },
1178         "name": "inputs_test.cwl",
1179     }
1180
1181     @stubs
1182     def test_inputs_empty(self, stubs):
1183         exited = arvados_cwl.main(
1184             ["--create-template",
1185              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1186             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1187         self.assertEqual(exited, 0)
1188
1189         stubs.api.pipeline_templates().create.assert_called_with(
1190             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1191
1192     @stubs
1193     def test_inputs(self, stubs):
1194         exited = arvados_cwl.main(
1195             ["--create-template",
1196              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1197             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1198         self.assertEqual(exited, 0)
1199
1200         expect_template = copy.deepcopy(self.expect_template)
1201         params = expect_template[
1202             "components"]["inputs_test.cwl"]["script_parameters"]
1203         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
1204         params["cwl:tool"] = '99999999999999999999999999999994+99/workflow.cwl#main'
1205         params["floatInput"]["value"] = 1.234
1206         params["boolInput"]["value"] = True
1207
1208         stubs.api.pipeline_templates().create.assert_called_with(
1209             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)