sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "nameext": ".txt",
 161                             "nameroot": "blorp",
 162                             "size": 16
 163                         }},
 164                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 165                               'listing': [
 166                                   {
 167                                       'basename': 'renamed.txt',
 168                                       'class': 'File', 'location':
 169                                       'keep:99999999999999999999999999999998+99/file1.txt',
 170                                       "nameext": ".txt",
 171                                       "nameroot": "renamed"
 172                                   }
 173                               ]}},
 174                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 175                         'arv:enable_reuse': True,
 176                         'arv:on_error': 'continue'
 177                     },
 178                     'repository': 'arvados',
 179                     'script_version': 'master',
 180                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 181                     'script': 'cwl-runner',
 182                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 183                 }
 184             }
 185         }
 186         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 187         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 188         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 189         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 190         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 191             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 192             "state": "Queued"
 193         }
 194         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 195         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 196
 197         with open("tests/wf/submit_wf_packed.cwl") as f:
 198             expect_packed_workflow = yaml.round_trip_load(f)
 199
 200         stubs.expect_container_spec = {
 201             'priority': 1,
 202             'mounts': {
 203                 '/var/spool/cwl': {
 204                     'writable': True,
 205                     'kind': 'collection'
 206                 },
 207                 '/var/lib/cwl/workflow.json': {
 208                     'content': expect_packed_workflow,
 209                     'kind': 'json'
 210                 },
 211                 'stdout': {
 212                     'path': '/var/spool/cwl/cwl.output.json',
 213                     'kind': 'file'
 214                 },
 215                 '/var/lib/cwl/cwl.input.json': {
 216                     'kind': 'json',
 217                     'content': {
 218                         'y': {
 219                             'basename': '99999999999999999999999999999998+99',
 220                             'location': 'keep:99999999999999999999999999999998+99',
 221                             'class': 'Directory'},
 222                         'x': {
 223                             'basename': u'blorp.txt',
 224                             'class': 'File',
 225                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 226                             "nameext": ".txt",
 227                             "nameroot": "blorp",
 228                             "size": 16
 229                         },
 230                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 231                             {'basename': 'renamed.txt',
 232                              'class': 'File',
 233                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 234                              "nameext": ".txt",
 235                              "nameroot": "renamed"
 236                             }
 237                         ]}
 238                     },
 239                     'kind': 'json'
 240                 }
 241             },
 242             'state': 'Committed',
 243             'owner_uuid': None,
 244             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 245                         '--enable-reuse', '--on-error=continue',
 246                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 247             'name': 'submit_wf.cwl',
 248             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 249             'output_path': '/var/spool/cwl',
 250             'cwd': '/var/spool/cwl',
 251             'runtime_constraints': {
 252                 'API': True,
 253                 'vcpus': 1,
 254                 'ram': 1024*1024*1024
 255             },
 256             "properties": {}
 257         }
 258
 259         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 260         stubs.api.workflows().create().execute.return_value = {
 261             "uuid": stubs.expect_workflow_uuid,
 262         }
 263         def update_mock(**kwargs):
 264             stubs.updated_uuid = kwargs.get('uuid')
 265             return mock.DEFAULT
 266         stubs.api.workflows().update.side_effect = update_mock
 267         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 268             "uuid": stubs.updated_uuid,
 269         }
 270
 271         return func(self, stubs, *args, **kwargs)
 272     return wrapped
 273
 274
 275 class TestSubmit(unittest.TestCase):
 276     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 277     @mock.patch("time.sleep")
 278     @stubs
 279     def test_submit(self, stubs, tm, arvdock):
 280         capture_stdout = cStringIO.StringIO()
 281         exited = arvados_cwl.main(
 282             ["--submit", "--no-wait", "--api=jobs", "--debug",
 283              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 284             capture_stdout, sys.stderr, api_client=stubs.api)
 285         self.assertEqual(exited, 0)
 286
 287         stubs.api.collections().create.assert_has_calls([
 288             mock.call(body=JsonDiffMatcher({
 289                 'manifest_text':
 290                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 291                 'replication_desired': None,
 292                 'name': 'submit_tool.cwl dependencies',
 293             }), ensure_unique_name=True),
 294             mock.call(body=JsonDiffMatcher({
 295                 'manifest_text':
 296                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 297                 'replication_desired': None,
 298                 'name': 'submit_wf.cwl input',
 299             }), ensure_unique_name=True),
 300             mock.call(body=JsonDiffMatcher({
 301                 'manifest_text':
 302                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 303                 'replication_desired': None,
 304                 'name': 'submit_wf.cwl',
 305             }), ensure_unique_name=True)        ])
 306
 307         arvdock.assert_has_calls([
 308             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 309             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 310         ])
 311
 312         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 313         stubs.api.pipeline_instances().create.assert_called_with(
 314             body=JsonDiffMatcher(expect_pipeline))
 315         self.assertEqual(capture_stdout.getvalue(),
 316                          stubs.expect_pipeline_uuid + '\n')
 317
 318
 319     @mock.patch("time.sleep")
 320     @stubs
 321     def test_submit_no_reuse(self, stubs, tm):
 322         capture_stdout = cStringIO.StringIO()
 323         exited = arvados_cwl.main(
 324             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 325              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 326             capture_stdout, sys.stderr, api_client=stubs.api)
 327         self.assertEqual(exited, 0)
 328
 329         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 330         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 331
 332         stubs.api.pipeline_instances().create.assert_called_with(
 333             body=JsonDiffMatcher(expect_pipeline))
 334         self.assertEqual(capture_stdout.getvalue(),
 335                          stubs.expect_pipeline_uuid + '\n')
 336
 337     @mock.patch("time.sleep")
 338     @stubs
 339     def test_submit_on_error(self, stubs, tm):
 340         capture_stdout = cStringIO.StringIO()
 341         exited = arvados_cwl.main(
 342             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 343              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 344             capture_stdout, sys.stderr, api_client=stubs.api)
 345         self.assertEqual(exited, 0)
 346
 347         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 348         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 349
 350         stubs.api.pipeline_instances().create.assert_called_with(
 351             body=JsonDiffMatcher(expect_pipeline))
 352         self.assertEqual(capture_stdout.getvalue(),
 353                          stubs.expect_pipeline_uuid + '\n')
 354
 355
 356     @mock.patch("time.sleep")
 357     @stubs
 358     def test_submit_runner_ram(self, stubs, tm):
 359         capture_stdout = cStringIO.StringIO()
 360         exited = arvados_cwl.main(
 361             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 362              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 363             capture_stdout, sys.stderr, api_client=stubs.api)
 364         self.assertEqual(exited, 0)
 365
 366         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 367         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 368
 369         stubs.api.pipeline_instances().create.assert_called_with(
 370             body=JsonDiffMatcher(expect_pipeline))
 371         self.assertEqual(capture_stdout.getvalue(),
 372                          stubs.expect_pipeline_uuid + '\n')
 373
 374
 375     @mock.patch("time.sleep")
 376     @stubs
 377     def test_submit_invalid_runner_ram(self, stubs, tm):
 378         capture_stdout = cStringIO.StringIO()
 379         exited = arvados_cwl.main(
 380             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 381              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 382             capture_stdout, sys.stderr, api_client=stubs.api)
 383         self.assertEqual(exited, 1)
 384
 385     @mock.patch("time.sleep")
 386     @stubs
 387     def test_submit_output_name(self, stubs, tm):
 388         output_name = "test_output_name"
 389
 390         capture_stdout = cStringIO.StringIO()
 391         exited = arvados_cwl.main(
 392             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 393              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 394             capture_stdout, sys.stderr, api_client=stubs.api)
 395         self.assertEqual(exited, 0)
 396
 397         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 398         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 399
 400         stubs.api.pipeline_instances().create.assert_called_with(
 401             body=JsonDiffMatcher(expect_pipeline))
 402         self.assertEqual(capture_stdout.getvalue(),
 403                          stubs.expect_pipeline_uuid + '\n')
 404
 405
 406     @mock.patch("time.sleep")
 407     @stubs
 408     def test_submit_pipeline_name(self, stubs, tm):
 409         capture_stdout = cStringIO.StringIO()
 410         exited = arvados_cwl.main(
 411             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 412              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 413             capture_stdout, sys.stderr, api_client=stubs.api)
 414         self.assertEqual(exited, 0)
 415
 416         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 417         expect_pipeline["name"] = "hello job 123"
 418
 419         stubs.api.pipeline_instances().create.assert_called_with(
 420             body=JsonDiffMatcher(expect_pipeline))
 421         self.assertEqual(capture_stdout.getvalue(),
 422                          stubs.expect_pipeline_uuid + '\n')
 423
 424     @mock.patch("time.sleep")
 425     @stubs
 426     def test_submit_output_tags(self, stubs, tm):
 427         output_tags = "tag0,tag1,tag2"
 428
 429         capture_stdout = cStringIO.StringIO()
 430         exited = arvados_cwl.main(
 431             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 432              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 433             capture_stdout, sys.stderr, api_client=stubs.api)
 434         self.assertEqual(exited, 0)
 435
 436         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 437         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 438
 439         stubs.api.pipeline_instances().create.assert_called_with(
 440             body=JsonDiffMatcher(expect_pipeline))
 441         self.assertEqual(capture_stdout.getvalue(),
 442                          stubs.expect_pipeline_uuid + '\n')
 443
 444     @mock.patch("time.sleep")
 445     @stubs
 446     def test_submit_with_project_uuid(self, stubs, tm):
 447         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 448
 449         exited = arvados_cwl.main(
 450             ["--submit", "--no-wait",
 451              "--project-uuid", project_uuid,
 452              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 453             sys.stdout, sys.stderr, api_client=stubs.api)
 454         self.assertEqual(exited, 0)
 455
 456         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 457         expect_pipeline["owner_uuid"] = project_uuid
 458         stubs.api.pipeline_instances().create.assert_called_with(
 459             body=JsonDiffMatcher(expect_pipeline))
 460
 461     @stubs
 462     def test_submit_container(self, stubs):
 463         capture_stdout = cStringIO.StringIO()
 464         try:
 465             exited = arvados_cwl.main(
 466                 ["--submit", "--no-wait", "--api=containers", "--debug",
 467                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 468                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 469             self.assertEqual(exited, 0)
 470         except:
 471             logging.exception("")
 472
 473         stubs.api.collections().create.assert_has_calls([
 474             mock.call(body=JsonDiffMatcher({
 475                 'manifest_text':
 476                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 477                 'replication_desired': None,
 478                 'name': 'submit_tool.cwl dependencies',
 479             }), ensure_unique_name=True),
 480             mock.call(body=JsonDiffMatcher({
 481                 'manifest_text':
 482                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 483                 'replication_desired': None,
 484                 'name': 'submit_wf.cwl input',
 485             }), ensure_unique_name=True)])
 486
 487         expect_container = copy.deepcopy(stubs.expect_container_spec)
 488         stubs.api.container_requests().create.assert_called_with(
 489             body=JsonDiffMatcher(expect_container))
 490         self.assertEqual(capture_stdout.getvalue(),
 491                          stubs.expect_container_request_uuid + '\n')
 492
 493     @stubs
 494     def test_submit_container_no_reuse(self, stubs):
 495         capture_stdout = cStringIO.StringIO()
 496         try:
 497             exited = arvados_cwl.main(
 498                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 499                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 500                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 501             self.assertEqual(exited, 0)
 502         except:
 503             logging.exception("")
 504
 505         expect_container = copy.deepcopy(stubs.expect_container_spec)
 506         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 507                                                   '--disable-reuse', '--on-error=continue',
 508                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 509
 510         stubs.api.container_requests().create.assert_called_with(
 511             body=JsonDiffMatcher(expect_container))
 512         self.assertEqual(capture_stdout.getvalue(),
 513                          stubs.expect_container_request_uuid + '\n')
 514
 515
 516     @stubs
 517     def test_submit_container_on_error(self, stubs):
 518         capture_stdout = cStringIO.StringIO()
 519         try:
 520             exited = arvados_cwl.main(
 521                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 522                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 523                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 524             self.assertEqual(exited, 0)
 525         except:
 526             logging.exception("")
 527
 528         expect_container = copy.deepcopy(stubs.expect_container_spec)
 529         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 530                                                   '--enable-reuse', '--on-error=stop',
 531                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 532
 533         stubs.api.container_requests().create.assert_called_with(
 534             body=JsonDiffMatcher(expect_container))
 535         self.assertEqual(capture_stdout.getvalue(),
 536                          stubs.expect_container_request_uuid + '\n')
 537
 538     @stubs
 539     def test_submit_container_output_name(self, stubs):
 540         output_name = "test_output_name"
 541
 542         capture_stdout = cStringIO.StringIO()
 543         try:
 544             exited = arvados_cwl.main(
 545                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 546                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 547                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 548             self.assertEqual(exited, 0)
 549         except:
 550             logging.exception("")
 551
 552         expect_container = copy.deepcopy(stubs.expect_container_spec)
 553         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 554                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 555                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 556         expect_container["output_name"] = output_name
 557
 558         stubs.api.container_requests().create.assert_called_with(
 559             body=JsonDiffMatcher(expect_container))
 560         self.assertEqual(capture_stdout.getvalue(),
 561                          stubs.expect_container_request_uuid + '\n')
 562
 563
 564     @stubs
 565     def test_submit_container_output_ttl(self, stubs):
 566         capture_stdout = cStringIO.StringIO()
 567         try:
 568             exited = arvados_cwl.main(
 569                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 570                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 571                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 572             self.assertEqual(exited, 0)
 573         except:
 574             logging.exception("")
 575
 576         expect_container = copy.deepcopy(stubs.expect_container_spec)
 577         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 578                                        '--enable-reuse', '--on-error=continue',
 579                                        "--intermediate-output-ttl=3600",
 580                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 581
 582         stubs.api.container_requests().create.assert_called_with(
 583             body=JsonDiffMatcher(expect_container))
 584         self.assertEqual(capture_stdout.getvalue(),
 585                          stubs.expect_container_request_uuid + '\n')
 586
 587     @stubs
 588     def test_submit_container_trash_intermediate(self, stubs):
 589         capture_stdout = cStringIO.StringIO()
 590         try:
 591             exited = arvados_cwl.main(
 592                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 593                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 594                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 595             self.assertEqual(exited, 0)
 596         except:
 597             logging.exception("")
 598
 599         expect_container = copy.deepcopy(stubs.expect_container_spec)
 600         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 601                                        '--enable-reuse', '--on-error=continue',
 602                                        "--trash-intermediate",
 603                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 604
 605         stubs.api.container_requests().create.assert_called_with(
 606             body=JsonDiffMatcher(expect_container))
 607         self.assertEqual(capture_stdout.getvalue(),
 608                          stubs.expect_container_request_uuid + '\n')
 609
 610     @stubs
 611     def test_submit_container_output_tags(self, stubs):
 612         output_tags = "tag0,tag1,tag2"
 613
 614         capture_stdout = cStringIO.StringIO()
 615         try:
 616             exited = arvados_cwl.main(
 617                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 618                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 619                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 620             self.assertEqual(exited, 0)
 621         except:
 622             logging.exception("")
 623
 624         expect_container = copy.deepcopy(stubs.expect_container_spec)
 625         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 626                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 627                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 628
 629         stubs.api.container_requests().create.assert_called_with(
 630             body=JsonDiffMatcher(expect_container))
 631         self.assertEqual(capture_stdout.getvalue(),
 632                          stubs.expect_container_request_uuid + '\n')
 633
 634     @stubs
 635     def test_submit_container_runner_ram(self, stubs):
 636         capture_stdout = cStringIO.StringIO()
 637         try:
 638             exited = arvados_cwl.main(
 639                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 640                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 641                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 642             self.assertEqual(exited, 0)
 643         except:
 644             logging.exception("")
 645
 646         expect_container = copy.deepcopy(stubs.expect_container_spec)
 647         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 648
 649         stubs.api.container_requests().create.assert_called_with(
 650             body=JsonDiffMatcher(expect_container))
 651         self.assertEqual(capture_stdout.getvalue(),
 652                          stubs.expect_container_request_uuid + '\n')
 653
 654     @mock.patch("arvados.collection.CollectionReader")
 655     @mock.patch("time.sleep")
 656     @stubs
 657     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 658         capture_stdout = cStringIO.StringIO()
 659         exited = arvados_cwl.main(
 660             ["--submit", "--no-wait", "--api=containers", "--debug",
 661              "tests/wf/submit_keepref_wf.cwl"],
 662             capture_stdout, sys.stderr, api_client=stubs.api)
 663         self.assertEqual(exited, 0)
 664
 665
 666     @mock.patch("arvados.collection.CollectionReader")
 667     @mock.patch("time.sleep")
 668     @stubs
 669     def test_submit_keepref(self, stubs, tm, reader):
 670         capture_stdout = cStringIO.StringIO()
 671
 672         with open("tests/wf/expect_arvworkflow.cwl") as f:
 673             reader().open().__enter__().read.return_value = f.read()
 674
 675         exited = arvados_cwl.main(
 676             ["--submit", "--no-wait", "--api=containers", "--debug",
 677              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 678             capture_stdout, sys.stderr, api_client=stubs.api)
 679         self.assertEqual(exited, 0)
 680
 681         expect_container = {
 682             'priority': 1,
 683             'mounts': {
 684                 '/var/spool/cwl': {
 685                     'writable': True,
 686                     'kind': 'collection'
 687                 },
 688                 'stdout': {
 689                     'path': '/var/spool/cwl/cwl.output.json',
 690                     'kind': 'file'
 691                 },
 692                 '/var/lib/cwl/workflow': {
 693                     'portable_data_hash': '99999999999999999999999999999994+99',
 694                     'kind': 'collection'
 695                 },
 696                 '/var/lib/cwl/cwl.input.json': {
 697                     'content': {
 698                         'x': 'XxX'
 699                     },
 700                     'kind': 'json'
 701                 }
 702             }, 'state': 'Committed',
 703             'owner_uuid': None,
 704             'output_path': '/var/spool/cwl',
 705             'name': 'expect_arvworkflow.cwl#main',
 706             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 707             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 708                         '--enable-reuse', '--on-error=continue',
 709                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 710             'cwd': '/var/spool/cwl',
 711             'runtime_constraints': {
 712                 'API': True,
 713                 'vcpus': 1,
 714                 'ram': 1073741824
 715             },
 716             "properties": {}
 717         }
 718
 719         stubs.api.container_requests().create.assert_called_with(
 720             body=JsonDiffMatcher(expect_container))
 721         self.assertEqual(capture_stdout.getvalue(),
 722                          stubs.expect_container_request_uuid + '\n')
 723
 724
 725     @mock.patch("arvados.collection.CollectionReader")
 726     @mock.patch("time.sleep")
 727     @stubs
 728     def test_submit_jobs_keepref(self, stubs, tm, reader):
 729         capture_stdout = cStringIO.StringIO()
 730
 731         with open("tests/wf/expect_arvworkflow.cwl") as f:
 732             reader().open().__enter__().read.return_value = f.read()
 733
 734         exited = arvados_cwl.main(
 735             ["--submit", "--no-wait", "--api=jobs", "--debug",
 736              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 737             capture_stdout, sys.stderr, api_client=stubs.api)
 738         self.assertEqual(exited, 0)
 739
 740         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 741         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 742         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 743         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 744         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 745         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 746         stubs.api.pipeline_instances().create.assert_called_with(
 747             body=JsonDiffMatcher(expect_pipeline))
 748
 749     @mock.patch("time.sleep")
 750     @stubs
 751     def test_submit_arvworkflow(self, stubs, tm):
 752         capture_stdout = cStringIO.StringIO()
 753
 754         with open("tests/wf/expect_arvworkflow.cwl") as f:
 755             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 756
 757         exited = arvados_cwl.main(
 758             ["--submit", "--no-wait", "--api=containers", "--debug",
 759              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 760             capture_stdout, sys.stderr, api_client=stubs.api)
 761         self.assertEqual(exited, 0)
 762
 763         expect_container = {
 764             'priority': 1,
 765             'mounts': {
 766                 '/var/spool/cwl': {
 767                     'writable': True,
 768                     'kind': 'collection'
 769                 },
 770                 'stdout': {
 771                     'path': '/var/spool/cwl/cwl.output.json',
 772                     'kind': 'file'
 773                 },
 774                 '/var/lib/cwl/workflow.json': {
 775                     'kind': 'json',
 776                     'content': {
 777                         'cwlVersion': 'v1.0',
 778                         '$graph': [
 779                             {
 780                                 'id': '#main',
 781                                 'inputs': [
 782                                     {'type': 'string', 'id': '#main/x'}
 783                                 ],
 784                                 'steps': [
 785                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 786                                      'run': '#submit_tool.cwl',
 787                                      'id': '#main/step1',
 788                                      'out': []}
 789                                 ],
 790                                 'class': 'Workflow',
 791                                 'outputs': []
 792                             },
 793                             {
 794                                 'inputs': [
 795                                     {
 796                                         'inputBinding': {'position': 1},
 797                                         'type': 'string',
 798                                         'id': '#submit_tool.cwl/x'}
 799                                 ],
 800                                 'requirements': [
 801                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 802                                 ],
 803                                 'id': '#submit_tool.cwl',
 804                                 'outputs': [],
 805                                 'baseCommand': 'cat',
 806                                 'class': 'CommandLineTool'
 807                             }
 808                         ]
 809                     }
 810                 },
 811                 '/var/lib/cwl/cwl.input.json': {
 812                     'content': {
 813                         'x': 'XxX'
 814                     },
 815                     'kind': 'json'
 816                 }
 817             }, 'state': 'Committed',
 818             'owner_uuid': None,
 819             'output_path': '/var/spool/cwl',
 820             'name': 'a test workflow',
 821             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 822             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 823                         '--enable-reuse', '--on-error=continue',
 824                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 825             'cwd': '/var/spool/cwl',
 826             'runtime_constraints': {
 827                 'API': True,
 828                 'vcpus': 1,
 829                 'ram': 1073741824
 830             },
 831             "properties": {
 832                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 833             }
 834         }
 835
 836         stubs.api.container_requests().create.assert_called_with(
 837             body=JsonDiffMatcher(expect_container))
 838         self.assertEqual(capture_stdout.getvalue(),
 839                          stubs.expect_container_request_uuid + '\n')
 840
 841
 842     @stubs
 843     def test_submit_container_name(self, stubs):
 844         capture_stdout = cStringIO.StringIO()
 845         try:
 846             exited = arvados_cwl.main(
 847                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 848                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 849                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 850             self.assertEqual(exited, 0)
 851         except:
 852             logging.exception("")
 853
 854         expect_container = copy.deepcopy(stubs.expect_container_spec)
 855         expect_container["name"] = "hello container 123"
 856
 857         stubs.api.container_requests().create.assert_called_with(
 858             body=JsonDiffMatcher(expect_container))
 859         self.assertEqual(capture_stdout.getvalue(),
 860                          stubs.expect_container_request_uuid + '\n')
 861
 862
 863     @stubs
 864     def test_submit_container_project(self, stubs):
 865         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 866         capture_stdout = cStringIO.StringIO()
 867         try:
 868             exited = arvados_cwl.main(
 869                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 870                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 871                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 872             self.assertEqual(exited, 0)
 873         except:
 874             logging.exception("")
 875
 876         expect_container = copy.deepcopy(stubs.expect_container_spec)
 877         expect_container["owner_uuid"] = project_uuid
 878         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 879                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid,
 880                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 881
 882         stubs.api.container_requests().create.assert_called_with(
 883             body=JsonDiffMatcher(expect_container))
 884         self.assertEqual(capture_stdout.getvalue(),
 885                          stubs.expect_container_request_uuid + '\n')
 886
 887
 888     @stubs
 889     def test_submit_job_runner_image(self, stubs):
 890         capture_stdout = cStringIO.StringIO()
 891         try:
 892             exited = arvados_cwl.main(
 893                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 894                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 895                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 896             self.assertEqual(exited, 0)
 897         except:
 898             logging.exception("")
 899
 900         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 901
 902         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 903         stubs.api.pipeline_instances().create.assert_called_with(
 904             body=JsonDiffMatcher(expect_pipeline))
 905         self.assertEqual(capture_stdout.getvalue(),
 906                          stubs.expect_pipeline_uuid + '\n')
 907
 908     @stubs
 909     def test_submit_container_runner_image(self, stubs):
 910         capture_stdout = cStringIO.StringIO()
 911         try:
 912             exited = arvados_cwl.main(
 913                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 914                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 915                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 916             self.assertEqual(exited, 0)
 917         except:
 918             logging.exception("")
 919
 920         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 921
 922         expect_container = copy.deepcopy(stubs.expect_container_spec)
 923         stubs.api.container_requests().create.assert_called_with(
 924             body=JsonDiffMatcher(expect_container))
 925         self.assertEqual(capture_stdout.getvalue(),
 926                          stubs.expect_container_request_uuid + '\n')
 927
 928
 929     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 930     @mock.patch("cwltool.docker.get_image")
 931     @mock.patch("arvados.api")
 932     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 933         arvrunner = mock.MagicMock()
 934         arvrunner.project_uuid = ""
 935         api.return_value = mock.MagicMock()
 936         arvrunner.api = api.return_value
 937         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 938                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 939                                                                         "link_class": "docker_image_repo+tag",
 940                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 941                                                                         "owner_uuid": "",
 942                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 943                                                             {"items": [{"created_at": "",
 944                                                                         "head_uuid": "",
 945                                                                         "link_class": "docker_image_hash",
 946                                                                         "name": "123456",
 947                                                                         "owner_uuid": "",
 948                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 949         )
 950         find_one_image_hash.return_value = "123456"
 951
 952         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 953                                                                               "owner_uuid": "",
 954                                                                               "manifest_text": "",
 955                                                                               "properties": ""
 956                                                                           }], "items_available": 1, "offset": 0},)
 957         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 958         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 959                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 960
 961 class TestCreateTemplate(unittest.TestCase):
 962     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 963
 964     def _adjust_script_params(self, expect_component):
 965         expect_component['script_parameters']['x'] = {
 966             'dataclass': 'File',
 967             'required': True,
 968             'type': 'File',
 969             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 970         }
 971         expect_component['script_parameters']['y'] = {
 972             'dataclass': 'Collection',
 973             'required': True,
 974             'type': 'Directory',
 975             'value': '99999999999999999999999999999998+99',
 976         }
 977         expect_component['script_parameters']['z'] = {
 978             'dataclass': 'Collection',
 979             'required': True,
 980             'type': 'Directory',
 981         }
 982
 983     @stubs
 984     def test_create(self, stubs):
 985         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 986
 987         capture_stdout = cStringIO.StringIO()
 988
 989         exited = arvados_cwl.main(
 990             ["--create-workflow", "--debug",
 991              "--api=jobs",
 992              "--project-uuid", project_uuid,
 993              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994             capture_stdout, sys.stderr, api_client=stubs.api)
 995         self.assertEqual(exited, 0)
 996
 997         stubs.api.pipeline_instances().create.refute_called()
 998         stubs.api.jobs().create.refute_called()
 999
1000         expect_component = copy.deepcopy(stubs.expect_job_spec)
1001         self._adjust_script_params(expect_component)
1002         expect_template = {
1003             "components": {
1004                 "submit_wf.cwl": expect_component,
1005             },
1006             "name": "submit_wf.cwl",
1007             "owner_uuid": project_uuid,
1008         }
1009         stubs.api.pipeline_templates().create.assert_called_with(
1010             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1011
1012         self.assertEqual(capture_stdout.getvalue(),
1013                          stubs.expect_pipeline_template_uuid + '\n')
1014
1015
1016     @stubs
1017     def test_create_name(self, stubs):
1018         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1019
1020         capture_stdout = cStringIO.StringIO()
1021
1022         exited = arvados_cwl.main(
1023             ["--create-workflow", "--debug",
1024              "--project-uuid", project_uuid,
1025              "--api=jobs",
1026              "--name", "testing 123",
1027              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1028             capture_stdout, sys.stderr, api_client=stubs.api)
1029         self.assertEqual(exited, 0)
1030
1031         stubs.api.pipeline_instances().create.refute_called()
1032         stubs.api.jobs().create.refute_called()
1033
1034         expect_component = copy.deepcopy(stubs.expect_job_spec)
1035         self._adjust_script_params(expect_component)
1036         expect_template = {
1037             "components": {
1038                 "testing 123": expect_component,
1039             },
1040             "name": "testing 123",
1041             "owner_uuid": project_uuid,
1042         }
1043         stubs.api.pipeline_templates().create.assert_called_with(
1044             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1045
1046         self.assertEqual(capture_stdout.getvalue(),
1047                          stubs.expect_pipeline_template_uuid + '\n')
1048
1049
1050     @stubs
1051     def test_update_name(self, stubs):
1052         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1053
1054         capture_stdout = cStringIO.StringIO()
1055
1056         exited = arvados_cwl.main(
1057             ["--update-workflow", self.existing_template_uuid,
1058              "--debug",
1059              "--project-uuid", project_uuid,
1060              "--api=jobs",
1061              "--name", "testing 123",
1062              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1063             capture_stdout, sys.stderr, api_client=stubs.api)
1064         self.assertEqual(exited, 0)
1065
1066         stubs.api.pipeline_instances().create.refute_called()
1067         stubs.api.jobs().create.refute_called()
1068
1069         expect_component = copy.deepcopy(stubs.expect_job_spec)
1070         self._adjust_script_params(expect_component)
1071         expect_template = {
1072             "components": {
1073                 "testing 123": expect_component,
1074             },
1075             "name": "testing 123",
1076             "owner_uuid": project_uuid,
1077         }
1078         stubs.api.pipeline_templates().create.refute_called()
1079         stubs.api.pipeline_templates().update.assert_called_with(
1080             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1081
1082         self.assertEqual(capture_stdout.getvalue(),
1083                          self.existing_template_uuid + '\n')
1084
1085
1086 class TestCreateWorkflow(unittest.TestCase):
1087     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1088     expect_workflow = StripYAMLComments(
1089         open("tests/wf/expect_packed.cwl").read())
1090
1091     @stubs
1092     def test_create(self, stubs):
1093         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1094
1095         capture_stdout = cStringIO.StringIO()
1096
1097         exited = arvados_cwl.main(
1098             ["--create-workflow", "--debug",
1099              "--api=containers",
1100              "--project-uuid", project_uuid,
1101              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1102             capture_stdout, sys.stderr, api_client=stubs.api)
1103         self.assertEqual(exited, 0)
1104
1105         stubs.api.pipeline_templates().create.refute_called()
1106         stubs.api.container_requests().create.refute_called()
1107
1108         body = {
1109             "workflow": {
1110                 "owner_uuid": project_uuid,
1111                 "name": "submit_wf.cwl",
1112                 "description": "",
1113                 "definition": self.expect_workflow,
1114             }
1115         }
1116         stubs.api.workflows().create.assert_called_with(
1117             body=JsonDiffMatcher(body))
1118
1119         self.assertEqual(capture_stdout.getvalue(),
1120                          stubs.expect_workflow_uuid + '\n')
1121
1122
1123     @stubs
1124     def test_create_name(self, stubs):
1125         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1126
1127         capture_stdout = cStringIO.StringIO()
1128
1129         exited = arvados_cwl.main(
1130             ["--create-workflow", "--debug",
1131              "--api=containers",
1132              "--project-uuid", project_uuid,
1133              "--name", "testing 123",
1134              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1135             capture_stdout, sys.stderr, api_client=stubs.api)
1136         self.assertEqual(exited, 0)
1137
1138         stubs.api.pipeline_templates().create.refute_called()
1139         stubs.api.container_requests().create.refute_called()
1140
1141         body = {
1142             "workflow": {
1143                 "owner_uuid": project_uuid,
1144                 "name": "testing 123",
1145                 "description": "",
1146                 "definition": self.expect_workflow,
1147             }
1148         }
1149         stubs.api.workflows().create.assert_called_with(
1150             body=JsonDiffMatcher(body))
1151
1152         self.assertEqual(capture_stdout.getvalue(),
1153                          stubs.expect_workflow_uuid + '\n')
1154
1155     @stubs
1156     def test_incompatible_api(self, stubs):
1157         capture_stderr = cStringIO.StringIO()
1158         logging.getLogger('arvados.cwl-runner').addHandler(
1159             logging.StreamHandler(capture_stderr))
1160
1161         exited = arvados_cwl.main(
1162             ["--update-workflow", self.existing_workflow_uuid,
1163              "--api=jobs",
1164              "--debug",
1165              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1166             sys.stderr, sys.stderr, api_client=stubs.api)
1167         self.assertEqual(exited, 1)
1168         self.assertRegexpMatches(
1169             capture_stderr.getvalue(),
1170             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1171
1172     @stubs
1173     def test_update(self, stubs):
1174         capture_stdout = cStringIO.StringIO()
1175
1176         exited = arvados_cwl.main(
1177             ["--update-workflow", self.existing_workflow_uuid,
1178              "--debug",
1179              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1180             capture_stdout, sys.stderr, api_client=stubs.api)
1181         self.assertEqual(exited, 0)
1182
1183         body = {
1184             "workflow": {
1185                 "name": "submit_wf.cwl",
1186                 "description": "",
1187                 "definition": self.expect_workflow,
1188             }
1189         }
1190         stubs.api.workflows().update.assert_called_with(
1191             uuid=self.existing_workflow_uuid,
1192             body=JsonDiffMatcher(body))
1193         self.assertEqual(capture_stdout.getvalue(),
1194                          self.existing_workflow_uuid + '\n')
1195
1196
1197     @stubs
1198     def test_update_name(self, stubs):
1199         capture_stdout = cStringIO.StringIO()
1200
1201         exited = arvados_cwl.main(
1202             ["--update-workflow", self.existing_workflow_uuid,
1203              "--debug", "--name", "testing 123",
1204              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1205             capture_stdout, sys.stderr, api_client=stubs.api)
1206         self.assertEqual(exited, 0)
1207
1208         body = {
1209             "workflow": {
1210                 "name": "testing 123",
1211                 "description": "",
1212                 "definition": self.expect_workflow,
1213             }
1214         }
1215         stubs.api.workflows().update.assert_called_with(
1216             uuid=self.existing_workflow_uuid,
1217             body=JsonDiffMatcher(body))
1218         self.assertEqual(capture_stdout.getvalue(),
1219                          self.existing_workflow_uuid + '\n')
1220
1221
1222     @stubs
1223     def test_create_collection_per_tool(self, stubs):
1224         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1225
1226         capture_stdout = cStringIO.StringIO()
1227
1228         exited = arvados_cwl.main(
1229             ["--create-workflow", "--debug",
1230              "--api=containers",
1231              "--project-uuid", project_uuid,
1232              "tests/collection_per_tool/collection_per_tool.cwl"],
1233             capture_stdout, sys.stderr, api_client=stubs.api)
1234         self.assertEqual(exited, 0)
1235
1236         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1237         expect_workflow = StripYAMLComments(open(toolfile).read())
1238
1239         body = {
1240             "workflow": {
1241                 "owner_uuid": project_uuid,
1242                 "name": "collection_per_tool.cwl",
1243                 "description": "",
1244                 "definition": expect_workflow,
1245             }
1246         }
1247         stubs.api.workflows().create.assert_called_with(
1248             body=JsonDiffMatcher(body))
1249
1250         self.assertEqual(capture_stdout.getvalue(),
1251                          stubs.expect_workflow_uuid + '\n')
1252
1253 class TestTemplateInputs(unittest.TestCase):
1254     expect_template = {
1255         "components": {
1256             "inputs_test.cwl": {
1257                 'runtime_constraints': {
1258                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1259                     'min_ram_mb_per_node': 1024
1260                 },
1261                 'script_parameters': {
1262                     'cwl:tool':
1263                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1264                     'optionalFloatInput': None,
1265                     'fileInput': {
1266                         'type': 'File',
1267                         'dataclass': 'File',
1268                         'required': True,
1269                         'title': "It's a file; we expect to find some characters in it.",
1270                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1271                     },
1272                     'floatInput': {
1273                         'type': 'float',
1274                         'dataclass': 'number',
1275                         'required': True,
1276                         'title': 'Floats like a duck',
1277                         'default': 0.1,
1278                         'value': 0.1,
1279                     },
1280                     'optionalFloatInput': {
1281                         'type': ['null', 'float'],
1282                         'dataclass': 'number',
1283                         'required': False,
1284                     },
1285                     'boolInput': {
1286                         'type': 'boolean',
1287                         'dataclass': 'boolean',
1288                         'required': True,
1289                         'title': 'True or false?',
1290                     },
1291                 },
1292                 'repository': 'arvados',
1293                 'script_version': 'master',
1294                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1295                 'script': 'cwl-runner',
1296             },
1297         },
1298         "name": "inputs_test.cwl",
1299     }
1300
1301     @stubs
1302     def test_inputs_empty(self, stubs):
1303         exited = arvados_cwl.main(
1304             ["--create-template",
1305              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1306             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1307         self.assertEqual(exited, 0)
1308
1309         stubs.api.pipeline_templates().create.assert_called_with(
1310             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1311
1312     @stubs
1313     def test_inputs(self, stubs):
1314         exited = arvados_cwl.main(
1315             ["--create-template",
1316              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1317             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1318         self.assertEqual(exited, 0)
1319
1320         expect_template = copy.deepcopy(self.expect_template)
1321         params = expect_template[
1322             "components"]["inputs_test.cwl"]["script_parameters"]
1323         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1324         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1325         params["floatInput"]["value"] = 1.234
1326         params["boolInput"]["value"] = True
1327
1328         stubs.api.pipeline_templates().create.assert_called_with(
1329             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)