sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 1,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'state': 'Committed',
 235             'owner_uuid': None,
 236             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 237                         '--enable-reuse', '--on-error=continue',
 238                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 239             'name': 'submit_wf.cwl',
 240             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 241             'output_path': '/var/spool/cwl',
 242             'cwd': '/var/spool/cwl',
 243             'runtime_constraints': {
 244                 'API': True,
 245                 'vcpus': 1,
 246                 'ram': 1024*1024*1024
 247             },
 248             'use_existing': True,
 249             'properties': {}
 250         }
 251
 252         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 253         stubs.api.workflows().create().execute.return_value = {
 254             "uuid": stubs.expect_workflow_uuid,
 255         }
 256         def update_mock(**kwargs):
 257             stubs.updated_uuid = kwargs.get('uuid')
 258             return mock.DEFAULT
 259         stubs.api.workflows().update.side_effect = update_mock
 260         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 261             "uuid": stubs.updated_uuid,
 262         }
 263
 264         return func(self, stubs, *args, **kwargs)
 265     return wrapped
 266
 267
 268 class TestSubmit(unittest.TestCase):
 269     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit(self, stubs, tm, arvdock):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--api=jobs", "--debug",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.api.collections().create.assert_has_calls([
 281             mock.call(body=JsonDiffMatcher({
 282                 'manifest_text':
 283                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 284                 'replication_desired': None,
 285                 'name': 'submit_tool.cwl dependencies',
 286             }), ensure_unique_name=True),
 287             mock.call(body=JsonDiffMatcher({
 288                 'manifest_text':
 289                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 290                 'replication_desired': None,
 291                 'name': 'submit_wf.cwl input',
 292             }), ensure_unique_name=True),
 293             mock.call(body=JsonDiffMatcher({
 294                 'manifest_text':
 295                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 296                 'replication_desired': None,
 297                 'name': 'submit_wf.cwl',
 298             }), ensure_unique_name=True)        ])
 299
 300         arvdock.assert_has_calls([
 301             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 302             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 303         ])
 304
 305         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 306         stubs.api.pipeline_instances().create.assert_called_with(
 307             body=JsonDiffMatcher(expect_pipeline))
 308         self.assertEqual(capture_stdout.getvalue(),
 309                          stubs.expect_pipeline_uuid + '\n')
 310
 311
 312     @mock.patch("time.sleep")
 313     @stubs
 314     def test_submit_no_reuse(self, stubs, tm):
 315         capture_stdout = cStringIO.StringIO()
 316         exited = arvados_cwl.main(
 317             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 318              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 319             capture_stdout, sys.stderr, api_client=stubs.api)
 320         self.assertEqual(exited, 0)
 321
 322         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 323         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 324         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331     @mock.patch("time.sleep")
 332     @stubs
 333     def test_submit_on_error(self, stubs, tm):
 334         capture_stdout = cStringIO.StringIO()
 335         exited = arvados_cwl.main(
 336             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 337              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 338             capture_stdout, sys.stderr, api_client=stubs.api)
 339         self.assertEqual(exited, 0)
 340
 341         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 342         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 343
 344         stubs.api.pipeline_instances().create.assert_called_with(
 345             body=JsonDiffMatcher(expect_pipeline))
 346         self.assertEqual(capture_stdout.getvalue(),
 347                          stubs.expect_pipeline_uuid + '\n')
 348
 349
 350     @mock.patch("time.sleep")
 351     @stubs
 352     def test_submit_runner_ram(self, stubs, tm):
 353         capture_stdout = cStringIO.StringIO()
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             capture_stdout, sys.stderr, api_client=stubs.api)
 358         self.assertEqual(exited, 0)
 359
 360         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 361         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 362
 363         stubs.api.pipeline_instances().create.assert_called_with(
 364             body=JsonDiffMatcher(expect_pipeline))
 365         self.assertEqual(capture_stdout.getvalue(),
 366                          stubs.expect_pipeline_uuid + '\n')
 367
 368
 369     @mock.patch("time.sleep")
 370     @stubs
 371     def test_submit_invalid_runner_ram(self, stubs, tm):
 372         capture_stdout = cStringIO.StringIO()
 373         exited = arvados_cwl.main(
 374             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 375              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 376             capture_stdout, sys.stderr, api_client=stubs.api)
 377         self.assertEqual(exited, 1)
 378
 379     @mock.patch("time.sleep")
 380     @stubs
 381     def test_submit_output_name(self, stubs, tm):
 382         output_name = "test_output_name"
 383
 384         capture_stdout = cStringIO.StringIO()
 385         exited = arvados_cwl.main(
 386             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 388             capture_stdout, sys.stderr, api_client=stubs.api)
 389         self.assertEqual(exited, 0)
 390
 391         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 392         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 393
 394         stubs.api.pipeline_instances().create.assert_called_with(
 395             body=JsonDiffMatcher(expect_pipeline))
 396         self.assertEqual(capture_stdout.getvalue(),
 397                          stubs.expect_pipeline_uuid + '\n')
 398
 399
 400     @mock.patch("time.sleep")
 401     @stubs
 402     def test_submit_pipeline_name(self, stubs, tm):
 403         capture_stdout = cStringIO.StringIO()
 404         exited = arvados_cwl.main(
 405             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 406              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 407             capture_stdout, sys.stderr, api_client=stubs.api)
 408         self.assertEqual(exited, 0)
 409
 410         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 411         expect_pipeline["name"] = "hello job 123"
 412
 413         stubs.api.pipeline_instances().create.assert_called_with(
 414             body=JsonDiffMatcher(expect_pipeline))
 415         self.assertEqual(capture_stdout.getvalue(),
 416                          stubs.expect_pipeline_uuid + '\n')
 417
 418     @mock.patch("time.sleep")
 419     @stubs
 420     def test_submit_output_tags(self, stubs, tm):
 421         output_tags = "tag0,tag1,tag2"
 422
 423         capture_stdout = cStringIO.StringIO()
 424         exited = arvados_cwl.main(
 425             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 426              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 427             capture_stdout, sys.stderr, api_client=stubs.api)
 428         self.assertEqual(exited, 0)
 429
 430         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 431         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 432
 433         stubs.api.pipeline_instances().create.assert_called_with(
 434             body=JsonDiffMatcher(expect_pipeline))
 435         self.assertEqual(capture_stdout.getvalue(),
 436                          stubs.expect_pipeline_uuid + '\n')
 437
 438     @mock.patch("time.sleep")
 439     @stubs
 440     def test_submit_with_project_uuid(self, stubs, tm):
 441         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 442
 443         exited = arvados_cwl.main(
 444             ["--submit", "--no-wait",
 445              "--project-uuid", project_uuid,
 446              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 447             sys.stdout, sys.stderr, api_client=stubs.api)
 448         self.assertEqual(exited, 0)
 449
 450         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 451         expect_pipeline["owner_uuid"] = project_uuid
 452         stubs.api.pipeline_instances().create.assert_called_with(
 453             body=JsonDiffMatcher(expect_pipeline))
 454
 455     @stubs
 456     def test_submit_container(self, stubs):
 457         capture_stdout = cStringIO.StringIO()
 458         try:
 459             exited = arvados_cwl.main(
 460                 ["--submit", "--no-wait", "--api=containers", "--debug",
 461                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 462                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 463             self.assertEqual(exited, 0)
 464         except:
 465             logging.exception("")
 466
 467         stubs.api.collections().create.assert_has_calls([
 468             mock.call(body=JsonDiffMatcher({
 469                 'manifest_text':
 470                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 471                 'replication_desired': None,
 472                 'name': 'submit_tool.cwl dependencies',
 473             }), ensure_unique_name=True),
 474             mock.call(body=JsonDiffMatcher({
 475                 'manifest_text':
 476                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 477                 'replication_desired': None,
 478                 'name': 'submit_wf.cwl input',
 479             }), ensure_unique_name=True)])
 480
 481         expect_container = copy.deepcopy(stubs.expect_container_spec)
 482         stubs.api.container_requests().create.assert_called_with(
 483             body=JsonDiffMatcher(expect_container))
 484         self.assertEqual(capture_stdout.getvalue(),
 485                          stubs.expect_container_request_uuid + '\n')
 486
 487     @stubs
 488     def test_submit_container_no_reuse(self, stubs):
 489         capture_stdout = cStringIO.StringIO()
 490         try:
 491             exited = arvados_cwl.main(
 492                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 493                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 494                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 495             self.assertEqual(exited, 0)
 496         except:
 497             logging.exception("")
 498
 499         expect_container = copy.deepcopy(stubs.expect_container_spec)
 500         expect_container["command"] = [
 501             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 502             '--disable-reuse', '--on-error=continue',
 503             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 504         expect_container["use_existing"] = False
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510
 511
 512     @stubs
 513     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 514         capture_stdout = cStringIO.StringIO()
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--api=containers", "--debug",
 518              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 519             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 520         self.assertEqual(exited, 0)
 521
 522         expect_container = copy.deepcopy(stubs.expect_container_spec)
 523         expect_container["command"] = [
 524             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 525             '--disable-reuse', '--on-error=continue',
 526             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 527         expect_container["use_existing"] = False
 528         expect_container["name"] = "submit_wf_no_reuse.cwl"
 529         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 530             {
 531                 "class": "http://arvados.org/cwl#ReuseRequirement",
 532                 "enableReuse": False,
 533             },
 534         ]
 535
 536         stubs.api.container_requests().create.assert_called_with(
 537             body=JsonDiffMatcher(expect_container))
 538         self.assertEqual(capture_stdout.getvalue(),
 539                          stubs.expect_container_request_uuid + '\n')
 540
 541
 542     @stubs
 543     def test_submit_container_on_error(self, stubs):
 544         capture_stdout = cStringIO.StringIO()
 545         try:
 546             exited = arvados_cwl.main(
 547                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 548                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 549                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 550             self.assertEqual(exited, 0)
 551         except:
 552             logging.exception("")
 553
 554         expect_container = copy.deepcopy(stubs.expect_container_spec)
 555         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 556                                                   '--enable-reuse', '--on-error=stop',
 557                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 558
 559         stubs.api.container_requests().create.assert_called_with(
 560             body=JsonDiffMatcher(expect_container))
 561         self.assertEqual(capture_stdout.getvalue(),
 562                          stubs.expect_container_request_uuid + '\n')
 563
 564     @stubs
 565     def test_submit_container_output_name(self, stubs):
 566         output_name = "test_output_name"
 567
 568         capture_stdout = cStringIO.StringIO()
 569         try:
 570             exited = arvados_cwl.main(
 571                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 572                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 573                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 574             self.assertEqual(exited, 0)
 575         except:
 576             logging.exception("")
 577
 578         expect_container = copy.deepcopy(stubs.expect_container_spec)
 579         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 580                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 581                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 582         expect_container["output_name"] = output_name
 583
 584         stubs.api.container_requests().create.assert_called_with(
 585             body=JsonDiffMatcher(expect_container))
 586         self.assertEqual(capture_stdout.getvalue(),
 587                          stubs.expect_container_request_uuid + '\n')
 588
 589
 590     @stubs
 591     def test_submit_container_output_ttl(self, stubs):
 592         capture_stdout = cStringIO.StringIO()
 593         try:
 594             exited = arvados_cwl.main(
 595                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 596                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598             self.assertEqual(exited, 0)
 599         except:
 600             logging.exception("")
 601
 602         expect_container = copy.deepcopy(stubs.expect_container_spec)
 603         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 604                                        '--enable-reuse', '--on-error=continue',
 605                                        "--intermediate-output-ttl=3600",
 606                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 607
 608         stubs.api.container_requests().create.assert_called_with(
 609             body=JsonDiffMatcher(expect_container))
 610         self.assertEqual(capture_stdout.getvalue(),
 611                          stubs.expect_container_request_uuid + '\n')
 612
 613     @stubs
 614     def test_submit_container_trash_intermediate(self, stubs):
 615         capture_stdout = cStringIO.StringIO()
 616         try:
 617             exited = arvados_cwl.main(
 618                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 619                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 620                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 621             self.assertEqual(exited, 0)
 622         except:
 623             logging.exception("")
 624
 625         expect_container = copy.deepcopy(stubs.expect_container_spec)
 626         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 627                                        '--enable-reuse', '--on-error=continue',
 628                                        "--trash-intermediate",
 629                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 630
 631         stubs.api.container_requests().create.assert_called_with(
 632             body=JsonDiffMatcher(expect_container))
 633         self.assertEqual(capture_stdout.getvalue(),
 634                          stubs.expect_container_request_uuid + '\n')
 635
 636     @stubs
 637     def test_submit_container_output_tags(self, stubs):
 638         output_tags = "tag0,tag1,tag2"
 639
 640         capture_stdout = cStringIO.StringIO()
 641         try:
 642             exited = arvados_cwl.main(
 643                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 644                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 645                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 646             self.assertEqual(exited, 0)
 647         except:
 648             logging.exception("")
 649
 650         expect_container = copy.deepcopy(stubs.expect_container_spec)
 651         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 652                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 653                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 654
 655         stubs.api.container_requests().create.assert_called_with(
 656             body=JsonDiffMatcher(expect_container))
 657         self.assertEqual(capture_stdout.getvalue(),
 658                          stubs.expect_container_request_uuid + '\n')
 659
 660     @stubs
 661     def test_submit_container_runner_ram(self, stubs):
 662         capture_stdout = cStringIO.StringIO()
 663         try:
 664             exited = arvados_cwl.main(
 665                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 666                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 667                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 668             self.assertEqual(exited, 0)
 669         except:
 670             logging.exception("")
 671
 672         expect_container = copy.deepcopy(stubs.expect_container_spec)
 673         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 674
 675         stubs.api.container_requests().create.assert_called_with(
 676             body=JsonDiffMatcher(expect_container))
 677         self.assertEqual(capture_stdout.getvalue(),
 678                          stubs.expect_container_request_uuid + '\n')
 679
 680     @mock.patch("arvados.collection.CollectionReader")
 681     @mock.patch("time.sleep")
 682     @stubs
 683     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 684         capture_stdout = cStringIO.StringIO()
 685         exited = arvados_cwl.main(
 686             ["--submit", "--no-wait", "--api=containers", "--debug",
 687              "tests/wf/submit_keepref_wf.cwl"],
 688             capture_stdout, sys.stderr, api_client=stubs.api)
 689         self.assertEqual(exited, 0)
 690
 691
 692     @mock.patch("arvados.collection.CollectionReader")
 693     @mock.patch("time.sleep")
 694     @stubs
 695     def test_submit_keepref(self, stubs, tm, reader):
 696         capture_stdout = cStringIO.StringIO()
 697
 698         with open("tests/wf/expect_arvworkflow.cwl") as f:
 699             reader().open().__enter__().read.return_value = f.read()
 700
 701         exited = arvados_cwl.main(
 702             ["--submit", "--no-wait", "--api=containers", "--debug",
 703              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 704             capture_stdout, sys.stderr, api_client=stubs.api)
 705         self.assertEqual(exited, 0)
 706
 707         expect_container = {
 708             'priority': 1,
 709             'mounts': {
 710                 '/var/spool/cwl': {
 711                     'writable': True,
 712                     'kind': 'collection'
 713                 },
 714                 'stdout': {
 715                     'path': '/var/spool/cwl/cwl.output.json',
 716                     'kind': 'file'
 717                 },
 718                 '/var/lib/cwl/workflow': {
 719                     'portable_data_hash': '99999999999999999999999999999994+99',
 720                     'kind': 'collection'
 721                 },
 722                 '/var/lib/cwl/cwl.input.json': {
 723                     'content': {
 724                         'x': 'XxX'
 725                     },
 726                     'kind': 'json'
 727                 }
 728             }, 'state': 'Committed',
 729             'owner_uuid': None,
 730             'output_path': '/var/spool/cwl',
 731             'name': 'expect_arvworkflow.cwl#main',
 732             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 733             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 734                         '--enable-reuse', '--on-error=continue',
 735                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 736             'cwd': '/var/spool/cwl',
 737             'runtime_constraints': {
 738                 'API': True,
 739                 'vcpus': 1,
 740                 'ram': 1073741824
 741             },
 742             'use_existing': True,
 743             'properties': {}
 744         }
 745
 746         stubs.api.container_requests().create.assert_called_with(
 747             body=JsonDiffMatcher(expect_container))
 748         self.assertEqual(capture_stdout.getvalue(),
 749                          stubs.expect_container_request_uuid + '\n')
 750
 751
 752     @mock.patch("arvados.collection.CollectionReader")
 753     @mock.patch("time.sleep")
 754     @stubs
 755     def test_submit_jobs_keepref(self, stubs, tm, reader):
 756         capture_stdout = cStringIO.StringIO()
 757
 758         with open("tests/wf/expect_arvworkflow.cwl") as f:
 759             reader().open().__enter__().read.return_value = f.read()
 760
 761         exited = arvados_cwl.main(
 762             ["--submit", "--no-wait", "--api=jobs", "--debug",
 763              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 764             capture_stdout, sys.stderr, api_client=stubs.api)
 765         self.assertEqual(exited, 0)
 766
 767         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 768         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 769         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 770         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 771         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 772         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 773         stubs.api.pipeline_instances().create.assert_called_with(
 774             body=JsonDiffMatcher(expect_pipeline))
 775
 776     @mock.patch("time.sleep")
 777     @stubs
 778     def test_submit_arvworkflow(self, stubs, tm):
 779         capture_stdout = cStringIO.StringIO()
 780
 781         with open("tests/wf/expect_arvworkflow.cwl") as f:
 782             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 783
 784         exited = arvados_cwl.main(
 785             ["--submit", "--no-wait", "--api=containers", "--debug",
 786              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 787             capture_stdout, sys.stderr, api_client=stubs.api)
 788         self.assertEqual(exited, 0)
 789
 790         expect_container = {
 791             'priority': 1,
 792             'mounts': {
 793                 '/var/spool/cwl': {
 794                     'writable': True,
 795                     'kind': 'collection'
 796                 },
 797                 'stdout': {
 798                     'path': '/var/spool/cwl/cwl.output.json',
 799                     'kind': 'file'
 800                 },
 801                 '/var/lib/cwl/workflow.json': {
 802                     'kind': 'json',
 803                     'content': {
 804                         'cwlVersion': 'v1.0',
 805                         '$graph': [
 806                             {
 807                                 'id': '#main',
 808                                 'inputs': [
 809                                     {'type': 'string', 'id': '#main/x'}
 810                                 ],
 811                                 'steps': [
 812                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 813                                      'run': '#submit_tool.cwl',
 814                                      'id': '#main/step1',
 815                                      'out': []}
 816                                 ],
 817                                 'class': 'Workflow',
 818                                 'outputs': []
 819                             },
 820                             {
 821                                 'inputs': [
 822                                     {
 823                                         'inputBinding': {'position': 1},
 824                                         'type': 'string',
 825                                         'id': '#submit_tool.cwl/x'}
 826                                 ],
 827                                 'requirements': [
 828                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 829                                 ],
 830                                 'id': '#submit_tool.cwl',
 831                                 'outputs': [],
 832                                 'baseCommand': 'cat',
 833                                 'class': 'CommandLineTool'
 834                             }
 835                         ]
 836                     }
 837                 },
 838                 '/var/lib/cwl/cwl.input.json': {
 839                     'content': {
 840                         'x': 'XxX'
 841                     },
 842                     'kind': 'json'
 843                 }
 844             }, 'state': 'Committed',
 845             'owner_uuid': None,
 846             'output_path': '/var/spool/cwl',
 847             'name': 'a test workflow',
 848             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 849             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 850                         '--enable-reuse', '--on-error=continue',
 851                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 852             'cwd': '/var/spool/cwl',
 853             'runtime_constraints': {
 854                 'API': True,
 855                 'vcpus': 1,
 856                 'ram': 1073741824
 857             },
 858             'use_existing': True,
 859             'properties': {
 860                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 861             }
 862         }
 863
 864         stubs.api.container_requests().create.assert_called_with(
 865             body=JsonDiffMatcher(expect_container))
 866         self.assertEqual(capture_stdout.getvalue(),
 867                          stubs.expect_container_request_uuid + '\n')
 868
 869
 870     @stubs
 871     def test_submit_container_name(self, stubs):
 872         capture_stdout = cStringIO.StringIO()
 873         try:
 874             exited = arvados_cwl.main(
 875                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 876                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 877                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 878             self.assertEqual(exited, 0)
 879         except:
 880             logging.exception("")
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["name"] = "hello container 123"
 884
 885         stubs.api.container_requests().create.assert_called_with(
 886             body=JsonDiffMatcher(expect_container))
 887         self.assertEqual(capture_stdout.getvalue(),
 888                          stubs.expect_container_request_uuid + '\n')
 889
 890
 891     @stubs
 892     def test_submit_container_project(self, stubs):
 893         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 894         capture_stdout = cStringIO.StringIO()
 895         try:
 896             exited = arvados_cwl.main(
 897                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 898                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 899                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 900             self.assertEqual(exited, 0)
 901         except:
 902             logging.exception("")
 903
 904         expect_container = copy.deepcopy(stubs.expect_container_spec)
 905         expect_container["owner_uuid"] = project_uuid
 906         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 907                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid,
 908                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 909
 910         stubs.api.container_requests().create.assert_called_with(
 911             body=JsonDiffMatcher(expect_container))
 912         self.assertEqual(capture_stdout.getvalue(),
 913                          stubs.expect_container_request_uuid + '\n')
 914
 915
 916     @stubs
 917     def test_submit_job_runner_image(self, stubs):
 918         capture_stdout = cStringIO.StringIO()
 919         try:
 920             exited = arvados_cwl.main(
 921                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 922                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 923                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 924             self.assertEqual(exited, 0)
 925         except:
 926             logging.exception("")
 927
 928         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 929
 930         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 931         stubs.api.pipeline_instances().create.assert_called_with(
 932             body=JsonDiffMatcher(expect_pipeline))
 933         self.assertEqual(capture_stdout.getvalue(),
 934                          stubs.expect_pipeline_uuid + '\n')
 935
 936     @stubs
 937     def test_submit_container_runner_image(self, stubs):
 938         capture_stdout = cStringIO.StringIO()
 939         try:
 940             exited = arvados_cwl.main(
 941                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 942                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 943                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 944             self.assertEqual(exited, 0)
 945         except:
 946             logging.exception("")
 947
 948         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 949
 950         expect_container = copy.deepcopy(stubs.expect_container_spec)
 951         stubs.api.container_requests().create.assert_called_with(
 952             body=JsonDiffMatcher(expect_container))
 953         self.assertEqual(capture_stdout.getvalue(),
 954                          stubs.expect_container_request_uuid + '\n')
 955
 956
 957     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 958     @mock.patch("cwltool.docker.get_image")
 959     @mock.patch("arvados.api")
 960     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 961         arvrunner = mock.MagicMock()
 962         arvrunner.project_uuid = ""
 963         api.return_value = mock.MagicMock()
 964         arvrunner.api = api.return_value
 965         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 966                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 967                                                                         "link_class": "docker_image_repo+tag",
 968                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 969                                                                         "owner_uuid": "",
 970                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 971                                                             {"items": [{"created_at": "",
 972                                                                         "head_uuid": "",
 973                                                                         "link_class": "docker_image_hash",
 974                                                                         "name": "123456",
 975                                                                         "owner_uuid": "",
 976                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 977         )
 978         find_one_image_hash.return_value = "123456"
 979
 980         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 981                                                                               "owner_uuid": "",
 982                                                                               "manifest_text": "",
 983                                                                               "properties": ""
 984                                                                           }], "items_available": 1, "offset": 0},)
 985         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 986         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 987                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 988
 989 class TestCreateTemplate(unittest.TestCase):
 990     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 991
 992     def _adjust_script_params(self, expect_component):
 993         expect_component['script_parameters']['x'] = {
 994             'dataclass': 'File',
 995             'required': True,
 996             'type': 'File',
 997             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 998         }
 999         expect_component['script_parameters']['y'] = {
1000             'dataclass': 'Collection',
1001             'required': True,
1002             'type': 'Directory',
1003             'value': '99999999999999999999999999999998+99',
1004         }
1005         expect_component['script_parameters']['z'] = {
1006             'dataclass': 'Collection',
1007             'required': True,
1008             'type': 'Directory',
1009         }
1010
1011     @stubs
1012     def test_create(self, stubs):
1013         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1014
1015         capture_stdout = cStringIO.StringIO()
1016
1017         exited = arvados_cwl.main(
1018             ["--create-workflow", "--debug",
1019              "--api=jobs",
1020              "--project-uuid", project_uuid,
1021              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1022             capture_stdout, sys.stderr, api_client=stubs.api)
1023         self.assertEqual(exited, 0)
1024
1025         stubs.api.pipeline_instances().create.refute_called()
1026         stubs.api.jobs().create.refute_called()
1027
1028         expect_component = copy.deepcopy(stubs.expect_job_spec)
1029         self._adjust_script_params(expect_component)
1030         expect_template = {
1031             "components": {
1032                 "submit_wf.cwl": expect_component,
1033             },
1034             "name": "submit_wf.cwl",
1035             "owner_uuid": project_uuid,
1036         }
1037         stubs.api.pipeline_templates().create.assert_called_with(
1038             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1039
1040         self.assertEqual(capture_stdout.getvalue(),
1041                          stubs.expect_pipeline_template_uuid + '\n')
1042
1043
1044     @stubs
1045     def test_create_name(self, stubs):
1046         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1047
1048         capture_stdout = cStringIO.StringIO()
1049
1050         exited = arvados_cwl.main(
1051             ["--create-workflow", "--debug",
1052              "--project-uuid", project_uuid,
1053              "--api=jobs",
1054              "--name", "testing 123",
1055              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1056             capture_stdout, sys.stderr, api_client=stubs.api)
1057         self.assertEqual(exited, 0)
1058
1059         stubs.api.pipeline_instances().create.refute_called()
1060         stubs.api.jobs().create.refute_called()
1061
1062         expect_component = copy.deepcopy(stubs.expect_job_spec)
1063         self._adjust_script_params(expect_component)
1064         expect_template = {
1065             "components": {
1066                 "testing 123": expect_component,
1067             },
1068             "name": "testing 123",
1069             "owner_uuid": project_uuid,
1070         }
1071         stubs.api.pipeline_templates().create.assert_called_with(
1072             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1073
1074         self.assertEqual(capture_stdout.getvalue(),
1075                          stubs.expect_pipeline_template_uuid + '\n')
1076
1077
1078     @stubs
1079     def test_update_name(self, stubs):
1080         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1081
1082         capture_stdout = cStringIO.StringIO()
1083
1084         exited = arvados_cwl.main(
1085             ["--update-workflow", self.existing_template_uuid,
1086              "--debug",
1087              "--project-uuid", project_uuid,
1088              "--api=jobs",
1089              "--name", "testing 123",
1090              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1091             capture_stdout, sys.stderr, api_client=stubs.api)
1092         self.assertEqual(exited, 0)
1093
1094         stubs.api.pipeline_instances().create.refute_called()
1095         stubs.api.jobs().create.refute_called()
1096
1097         expect_component = copy.deepcopy(stubs.expect_job_spec)
1098         self._adjust_script_params(expect_component)
1099         expect_template = {
1100             "components": {
1101                 "testing 123": expect_component,
1102             },
1103             "name": "testing 123",
1104             "owner_uuid": project_uuid,
1105         }
1106         stubs.api.pipeline_templates().create.refute_called()
1107         stubs.api.pipeline_templates().update.assert_called_with(
1108             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1109
1110         self.assertEqual(capture_stdout.getvalue(),
1111                          self.existing_template_uuid + '\n')
1112
1113
1114 class TestCreateWorkflow(unittest.TestCase):
1115     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1116     expect_workflow = StripYAMLComments(
1117         open("tests/wf/expect_packed.cwl").read())
1118
1119     @stubs
1120     def test_create(self, stubs):
1121         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1122
1123         capture_stdout = cStringIO.StringIO()
1124
1125         exited = arvados_cwl.main(
1126             ["--create-workflow", "--debug",
1127              "--api=containers",
1128              "--project-uuid", project_uuid,
1129              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1130             capture_stdout, sys.stderr, api_client=stubs.api)
1131         self.assertEqual(exited, 0)
1132
1133         stubs.api.pipeline_templates().create.refute_called()
1134         stubs.api.container_requests().create.refute_called()
1135
1136         body = {
1137             "workflow": {
1138                 "owner_uuid": project_uuid,
1139                 "name": "submit_wf.cwl",
1140                 "description": "",
1141                 "definition": self.expect_workflow,
1142             }
1143         }
1144         stubs.api.workflows().create.assert_called_with(
1145             body=JsonDiffMatcher(body))
1146
1147         self.assertEqual(capture_stdout.getvalue(),
1148                          stubs.expect_workflow_uuid + '\n')
1149
1150
1151     @stubs
1152     def test_create_name(self, stubs):
1153         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1154
1155         capture_stdout = cStringIO.StringIO()
1156
1157         exited = arvados_cwl.main(
1158             ["--create-workflow", "--debug",
1159              "--api=containers",
1160              "--project-uuid", project_uuid,
1161              "--name", "testing 123",
1162              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1163             capture_stdout, sys.stderr, api_client=stubs.api)
1164         self.assertEqual(exited, 0)
1165
1166         stubs.api.pipeline_templates().create.refute_called()
1167         stubs.api.container_requests().create.refute_called()
1168
1169         body = {
1170             "workflow": {
1171                 "owner_uuid": project_uuid,
1172                 "name": "testing 123",
1173                 "description": "",
1174                 "definition": self.expect_workflow,
1175             }
1176         }
1177         stubs.api.workflows().create.assert_called_with(
1178             body=JsonDiffMatcher(body))
1179
1180         self.assertEqual(capture_stdout.getvalue(),
1181                          stubs.expect_workflow_uuid + '\n')
1182
1183     @stubs
1184     def test_incompatible_api(self, stubs):
1185         capture_stderr = cStringIO.StringIO()
1186         logging.getLogger('arvados.cwl-runner').addHandler(
1187             logging.StreamHandler(capture_stderr))
1188
1189         exited = arvados_cwl.main(
1190             ["--update-workflow", self.existing_workflow_uuid,
1191              "--api=jobs",
1192              "--debug",
1193              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1194             sys.stderr, sys.stderr, api_client=stubs.api)
1195         self.assertEqual(exited, 1)
1196         self.assertRegexpMatches(
1197             capture_stderr.getvalue(),
1198             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1199
1200     @stubs
1201     def test_update(self, stubs):
1202         capture_stdout = cStringIO.StringIO()
1203
1204         exited = arvados_cwl.main(
1205             ["--update-workflow", self.existing_workflow_uuid,
1206              "--debug",
1207              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1208             capture_stdout, sys.stderr, api_client=stubs.api)
1209         self.assertEqual(exited, 0)
1210
1211         body = {
1212             "workflow": {
1213                 "name": "submit_wf.cwl",
1214                 "description": "",
1215                 "definition": self.expect_workflow,
1216             }
1217         }
1218         stubs.api.workflows().update.assert_called_with(
1219             uuid=self.existing_workflow_uuid,
1220             body=JsonDiffMatcher(body))
1221         self.assertEqual(capture_stdout.getvalue(),
1222                          self.existing_workflow_uuid + '\n')
1223
1224
1225     @stubs
1226     def test_update_name(self, stubs):
1227         capture_stdout = cStringIO.StringIO()
1228
1229         exited = arvados_cwl.main(
1230             ["--update-workflow", self.existing_workflow_uuid,
1231              "--debug", "--name", "testing 123",
1232              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1233             capture_stdout, sys.stderr, api_client=stubs.api)
1234         self.assertEqual(exited, 0)
1235
1236         body = {
1237             "workflow": {
1238                 "name": "testing 123",
1239                 "description": "",
1240                 "definition": self.expect_workflow,
1241             }
1242         }
1243         stubs.api.workflows().update.assert_called_with(
1244             uuid=self.existing_workflow_uuid,
1245             body=JsonDiffMatcher(body))
1246         self.assertEqual(capture_stdout.getvalue(),
1247                          self.existing_workflow_uuid + '\n')
1248
1249
1250     @stubs
1251     def test_create_collection_per_tool(self, stubs):
1252         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1253
1254         capture_stdout = cStringIO.StringIO()
1255
1256         exited = arvados_cwl.main(
1257             ["--create-workflow", "--debug",
1258              "--api=containers",
1259              "--project-uuid", project_uuid,
1260              "tests/collection_per_tool/collection_per_tool.cwl"],
1261             capture_stdout, sys.stderr, api_client=stubs.api)
1262         self.assertEqual(exited, 0)
1263
1264         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1265         expect_workflow = StripYAMLComments(open(toolfile).read())
1266
1267         body = {
1268             "workflow": {
1269                 "owner_uuid": project_uuid,
1270                 "name": "collection_per_tool.cwl",
1271                 "description": "",
1272                 "definition": expect_workflow,
1273             }
1274         }
1275         stubs.api.workflows().create.assert_called_with(
1276             body=JsonDiffMatcher(body))
1277
1278         self.assertEqual(capture_stdout.getvalue(),
1279                          stubs.expect_workflow_uuid + '\n')
1280
1281 class TestTemplateInputs(unittest.TestCase):
1282     expect_template = {
1283         "components": {
1284             "inputs_test.cwl": {
1285                 'runtime_constraints': {
1286                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1287                     'min_ram_mb_per_node': 1024
1288                 },
1289                 'script_parameters': {
1290                     'cwl:tool':
1291                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1292                     'optionalFloatInput': None,
1293                     'fileInput': {
1294                         'type': 'File',
1295                         'dataclass': 'File',
1296                         'required': True,
1297                         'title': "It's a file; we expect to find some characters in it.",
1298                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1299                     },
1300                     'floatInput': {
1301                         'type': 'float',
1302                         'dataclass': 'number',
1303                         'required': True,
1304                         'title': 'Floats like a duck',
1305                         'default': 0.1,
1306                         'value': 0.1,
1307                     },
1308                     'optionalFloatInput': {
1309                         'type': ['null', 'float'],
1310                         'dataclass': 'number',
1311                         'required': False,
1312                     },
1313                     'boolInput': {
1314                         'type': 'boolean',
1315                         'dataclass': 'boolean',
1316                         'required': True,
1317                         'title': 'True or false?',
1318                     },
1319                 },
1320                 'repository': 'arvados',
1321                 'script_version': 'master',
1322                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1323                 'script': 'cwl-runner',
1324             },
1325         },
1326         "name": "inputs_test.cwl",
1327     }
1328
1329     @stubs
1330     def test_inputs_empty(self, stubs):
1331         exited = arvados_cwl.main(
1332             ["--create-template",
1333              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1334             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1335         self.assertEqual(exited, 0)
1336
1337         stubs.api.pipeline_templates().create.assert_called_with(
1338             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1339
1340     @stubs
1341     def test_inputs(self, stubs):
1342         exited = arvados_cwl.main(
1343             ["--create-template",
1344              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1345             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1346         self.assertEqual(exited, 0)
1347
1348         expect_template = copy.deepcopy(self.expect_template)
1349         params = expect_template[
1350             "components"]["inputs_test.cwl"]["script_parameters"]
1351         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1352         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1353         params["floatInput"]["value"] = 1.234
1354         params["boolInput"]["value"] = True
1355
1356         stubs.api.pipeline_templates().create.assert_called_with(
1357             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)