sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 1,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'state': 'Committed',
 235             'owner_uuid': None,
 236             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 237                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 238                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 239             'name': 'submit_wf.cwl',
 240             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 241             'output_path': '/var/spool/cwl',
 242             'cwd': '/var/spool/cwl',
 243             'runtime_constraints': {
 244                 'API': True,
 245                 'vcpus': 1,
 246                 'ram': 1024*1024*1024
 247             },
 248             'use_existing': True,
 249             'properties': {}
 250         }
 251
 252         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 253         stubs.api.workflows().create().execute.return_value = {
 254             "uuid": stubs.expect_workflow_uuid,
 255         }
 256         def update_mock(**kwargs):
 257             stubs.updated_uuid = kwargs.get('uuid')
 258             return mock.DEFAULT
 259         stubs.api.workflows().update.side_effect = update_mock
 260         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 261             "uuid": stubs.updated_uuid,
 262         }
 263
 264         return func(self, stubs, *args, **kwargs)
 265     return wrapped
 266
 267
 268 class TestSubmit(unittest.TestCase):
 269     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit(self, stubs, tm, arvdock):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--api=jobs", "--debug",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.api.collections().create.assert_has_calls([
 281             mock.call(body=JsonDiffMatcher({
 282                 'manifest_text':
 283                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 284                 'replication_desired': None,
 285                 'name': 'submit_tool.cwl dependencies',
 286             }), ensure_unique_name=True),
 287             mock.call(body=JsonDiffMatcher({
 288                 'manifest_text':
 289                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 290                 'replication_desired': None,
 291                 'name': 'submit_wf.cwl input',
 292             }), ensure_unique_name=True),
 293             mock.call(body=JsonDiffMatcher({
 294                 'manifest_text':
 295                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 296                 'replication_desired': None,
 297                 'name': 'submit_wf.cwl',
 298             }), ensure_unique_name=True)        ])
 299
 300         arvdock.assert_has_calls([
 301             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 302             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 303         ])
 304
 305         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 306         stubs.api.pipeline_instances().create.assert_called_with(
 307             body=JsonDiffMatcher(expect_pipeline))
 308         self.assertEqual(capture_stdout.getvalue(),
 309                          stubs.expect_pipeline_uuid + '\n')
 310
 311
 312     @mock.patch("time.sleep")
 313     @stubs
 314     def test_submit_no_reuse(self, stubs, tm):
 315         capture_stdout = cStringIO.StringIO()
 316         exited = arvados_cwl.main(
 317             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 318              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 319             capture_stdout, sys.stderr, api_client=stubs.api)
 320         self.assertEqual(exited, 0)
 321
 322         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 323         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 324         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331     @mock.patch("time.sleep")
 332     @stubs
 333     def test_submit_on_error(self, stubs, tm):
 334         capture_stdout = cStringIO.StringIO()
 335         exited = arvados_cwl.main(
 336             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 337              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 338             capture_stdout, sys.stderr, api_client=stubs.api)
 339         self.assertEqual(exited, 0)
 340
 341         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 342         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 343
 344         stubs.api.pipeline_instances().create.assert_called_with(
 345             body=JsonDiffMatcher(expect_pipeline))
 346         self.assertEqual(capture_stdout.getvalue(),
 347                          stubs.expect_pipeline_uuid + '\n')
 348
 349
 350     @mock.patch("time.sleep")
 351     @stubs
 352     def test_submit_runner_ram(self, stubs, tm):
 353         capture_stdout = cStringIO.StringIO()
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             capture_stdout, sys.stderr, api_client=stubs.api)
 358         self.assertEqual(exited, 0)
 359
 360         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 361         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 362
 363         stubs.api.pipeline_instances().create.assert_called_with(
 364             body=JsonDiffMatcher(expect_pipeline))
 365         self.assertEqual(capture_stdout.getvalue(),
 366                          stubs.expect_pipeline_uuid + '\n')
 367
 368
 369     @mock.patch("time.sleep")
 370     @stubs
 371     def test_submit_invalid_runner_ram(self, stubs, tm):
 372         capture_stdout = cStringIO.StringIO()
 373         exited = arvados_cwl.main(
 374             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 375              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 376             capture_stdout, sys.stderr, api_client=stubs.api)
 377         self.assertEqual(exited, 1)
 378
 379     @mock.patch("time.sleep")
 380     @stubs
 381     def test_submit_output_name(self, stubs, tm):
 382         output_name = "test_output_name"
 383
 384         capture_stdout = cStringIO.StringIO()
 385         exited = arvados_cwl.main(
 386             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 388             capture_stdout, sys.stderr, api_client=stubs.api)
 389         self.assertEqual(exited, 0)
 390
 391         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 392         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 393
 394         stubs.api.pipeline_instances().create.assert_called_with(
 395             body=JsonDiffMatcher(expect_pipeline))
 396         self.assertEqual(capture_stdout.getvalue(),
 397                          stubs.expect_pipeline_uuid + '\n')
 398
 399
 400     @mock.patch("time.sleep")
 401     @stubs
 402     def test_submit_pipeline_name(self, stubs, tm):
 403         capture_stdout = cStringIO.StringIO()
 404         exited = arvados_cwl.main(
 405             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 406              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 407             capture_stdout, sys.stderr, api_client=stubs.api)
 408         self.assertEqual(exited, 0)
 409
 410         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 411         expect_pipeline["name"] = "hello job 123"
 412
 413         stubs.api.pipeline_instances().create.assert_called_with(
 414             body=JsonDiffMatcher(expect_pipeline))
 415         self.assertEqual(capture_stdout.getvalue(),
 416                          stubs.expect_pipeline_uuid + '\n')
 417
 418     @mock.patch("time.sleep")
 419     @stubs
 420     def test_submit_output_tags(self, stubs, tm):
 421         output_tags = "tag0,tag1,tag2"
 422
 423         capture_stdout = cStringIO.StringIO()
 424         exited = arvados_cwl.main(
 425             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 426              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 427             capture_stdout, sys.stderr, api_client=stubs.api)
 428         self.assertEqual(exited, 0)
 429
 430         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 431         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 432
 433         stubs.api.pipeline_instances().create.assert_called_with(
 434             body=JsonDiffMatcher(expect_pipeline))
 435         self.assertEqual(capture_stdout.getvalue(),
 436                          stubs.expect_pipeline_uuid + '\n')
 437
 438     @mock.patch("time.sleep")
 439     @stubs
 440     def test_submit_with_project_uuid(self, stubs, tm):
 441         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 442
 443         exited = arvados_cwl.main(
 444             ["--submit", "--no-wait",
 445              "--project-uuid", project_uuid,
 446              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 447             sys.stdout, sys.stderr, api_client=stubs.api)
 448         self.assertEqual(exited, 0)
 449
 450         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 451         expect_pipeline["owner_uuid"] = project_uuid
 452         stubs.api.pipeline_instances().create.assert_called_with(
 453             body=JsonDiffMatcher(expect_pipeline))
 454
 455     @stubs
 456     def test_submit_container(self, stubs):
 457         capture_stdout = cStringIO.StringIO()
 458         try:
 459             exited = arvados_cwl.main(
 460                 ["--submit", "--no-wait", "--api=containers", "--debug",
 461                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 462                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 463             self.assertEqual(exited, 0)
 464         except:
 465             logging.exception("")
 466
 467         stubs.api.collections().create.assert_has_calls([
 468             mock.call(body=JsonDiffMatcher({
 469                 'manifest_text':
 470                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 471                 'replication_desired': None,
 472                 'name': 'submit_tool.cwl dependencies',
 473             }), ensure_unique_name=True),
 474             mock.call(body=JsonDiffMatcher({
 475                 'manifest_text':
 476                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 477                 'replication_desired': None,
 478                 'name': 'submit_wf.cwl input',
 479             }), ensure_unique_name=True)])
 480
 481         expect_container = copy.deepcopy(stubs.expect_container_spec)
 482         stubs.api.container_requests().create.assert_called_with(
 483             body=JsonDiffMatcher(expect_container))
 484         self.assertEqual(capture_stdout.getvalue(),
 485                          stubs.expect_container_request_uuid + '\n')
 486
 487     @stubs
 488     def test_submit_container_no_reuse(self, stubs):
 489         capture_stdout = cStringIO.StringIO()
 490         try:
 491             exited = arvados_cwl.main(
 492                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 493                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 494                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 495             self.assertEqual(exited, 0)
 496         except:
 497             logging.exception("")
 498
 499         expect_container = copy.deepcopy(stubs.expect_container_spec)
 500         expect_container["command"] = [
 501             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 502             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 503             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 504         expect_container["use_existing"] = False
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510
 511
 512     @stubs
 513     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 514         capture_stdout = cStringIO.StringIO()
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--api=containers", "--debug",
 518              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 519             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 520         self.assertEqual(exited, 0)
 521
 522         expect_container = copy.deepcopy(stubs.expect_container_spec)
 523         expect_container["command"] = [
 524             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 525             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 526             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 527         expect_container["use_existing"] = False
 528         expect_container["name"] = "submit_wf_no_reuse.cwl"
 529         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 530             {
 531                 "class": "http://arvados.org/cwl#ReuseRequirement",
 532                 "enableReuse": False,
 533             },
 534         ]
 535         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 536             "arv": "http://arvados.org/cwl#",
 537             "cwltool": "http://commonwl.org/cwltool#"
 538         }
 539
 540         stubs.api.container_requests().create.assert_called_with(
 541             body=JsonDiffMatcher(expect_container))
 542         self.assertEqual(capture_stdout.getvalue(),
 543                          stubs.expect_container_request_uuid + '\n')
 544
 545
 546     @stubs
 547     def test_submit_container_on_error(self, stubs):
 548         capture_stdout = cStringIO.StringIO()
 549         try:
 550             exited = arvados_cwl.main(
 551                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 552                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 553                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 554             self.assertEqual(exited, 0)
 555         except:
 556             logging.exception("")
 557
 558         expect_container = copy.deepcopy(stubs.expect_container_spec)
 559         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 560                                                   '--enable-reuse', '--on-error=stop', '--eval-timeout=20',
 561                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 562
 563         stubs.api.container_requests().create.assert_called_with(
 564             body=JsonDiffMatcher(expect_container))
 565         self.assertEqual(capture_stdout.getvalue(),
 566                          stubs.expect_container_request_uuid + '\n')
 567
 568     @stubs
 569     def test_submit_container_output_name(self, stubs):
 570         output_name = "test_output_name"
 571
 572         capture_stdout = cStringIO.StringIO()
 573         try:
 574             exited = arvados_cwl.main(
 575                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 576                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 577                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 578             self.assertEqual(exited, 0)
 579         except:
 580             logging.exception("")
 581
 582         expect_container = copy.deepcopy(stubs.expect_container_spec)
 583         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 584                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 585                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586         expect_container["output_name"] = output_name
 587
 588         stubs.api.container_requests().create.assert_called_with(
 589             body=JsonDiffMatcher(expect_container))
 590         self.assertEqual(capture_stdout.getvalue(),
 591                          stubs.expect_container_request_uuid + '\n')
 592
 593
 594     @stubs
 595     def test_submit_container_output_ttl(self, stubs):
 596         capture_stdout = cStringIO.StringIO()
 597         try:
 598             exited = arvados_cwl.main(
 599                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 600                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 601                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 602             self.assertEqual(exited, 0)
 603         except:
 604             logging.exception("")
 605
 606         expect_container = copy.deepcopy(stubs.expect_container_spec)
 607         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 608                                        '--enable-reuse', '--on-error=continue',
 609                                        "--intermediate-output-ttl=3600", '--eval-timeout=20',
 610                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 611
 612         stubs.api.container_requests().create.assert_called_with(
 613             body=JsonDiffMatcher(expect_container))
 614         self.assertEqual(capture_stdout.getvalue(),
 615                          stubs.expect_container_request_uuid + '\n')
 616
 617     @stubs
 618     def test_submit_container_trash_intermediate(self, stubs):
 619         capture_stdout = cStringIO.StringIO()
 620         try:
 621             exited = arvados_cwl.main(
 622                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 623                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 624                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 625             self.assertEqual(exited, 0)
 626         except:
 627             logging.exception("")
 628
 629         expect_container = copy.deepcopy(stubs.expect_container_spec)
 630         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 631                                        '--enable-reuse', '--on-error=continue',
 632                                        "--trash-intermediate", '--eval-timeout=20',
 633                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 634
 635         stubs.api.container_requests().create.assert_called_with(
 636             body=JsonDiffMatcher(expect_container))
 637         self.assertEqual(capture_stdout.getvalue(),
 638                          stubs.expect_container_request_uuid + '\n')
 639
 640     @stubs
 641     def test_submit_container_output_tags(self, stubs):
 642         output_tags = "tag0,tag1,tag2"
 643
 644         capture_stdout = cStringIO.StringIO()
 645         try:
 646             exited = arvados_cwl.main(
 647                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 648                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 649                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 650             self.assertEqual(exited, 0)
 651         except:
 652             logging.exception("")
 653
 654         expect_container = copy.deepcopy(stubs.expect_container_spec)
 655         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 656                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 657                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 658
 659         stubs.api.container_requests().create.assert_called_with(
 660             body=JsonDiffMatcher(expect_container))
 661         self.assertEqual(capture_stdout.getvalue(),
 662                          stubs.expect_container_request_uuid + '\n')
 663
 664     @stubs
 665     def test_submit_container_runner_ram(self, stubs):
 666         capture_stdout = cStringIO.StringIO()
 667         try:
 668             exited = arvados_cwl.main(
 669                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 670                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 671                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 672             self.assertEqual(exited, 0)
 673         except:
 674             logging.exception("")
 675
 676         expect_container = copy.deepcopy(stubs.expect_container_spec)
 677         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 678
 679         stubs.api.container_requests().create.assert_called_with(
 680             body=JsonDiffMatcher(expect_container))
 681         self.assertEqual(capture_stdout.getvalue(),
 682                          stubs.expect_container_request_uuid + '\n')
 683
 684     @mock.patch("arvados.collection.CollectionReader")
 685     @mock.patch("time.sleep")
 686     @stubs
 687     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 688         capture_stdout = cStringIO.StringIO()
 689         exited = arvados_cwl.main(
 690             ["--submit", "--no-wait", "--api=containers", "--debug",
 691              "tests/wf/submit_keepref_wf.cwl"],
 692             capture_stdout, sys.stderr, api_client=stubs.api)
 693         self.assertEqual(exited, 0)
 694
 695
 696     @mock.patch("arvados.collection.CollectionReader")
 697     @mock.patch("time.sleep")
 698     @stubs
 699     def test_submit_keepref(self, stubs, tm, reader):
 700         capture_stdout = cStringIO.StringIO()
 701
 702         with open("tests/wf/expect_arvworkflow.cwl") as f:
 703             reader().open().__enter__().read.return_value = f.read()
 704
 705         exited = arvados_cwl.main(
 706             ["--submit", "--no-wait", "--api=containers", "--debug",
 707              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 708             capture_stdout, sys.stderr, api_client=stubs.api)
 709         self.assertEqual(exited, 0)
 710
 711         expect_container = {
 712             'priority': 1,
 713             'mounts': {
 714                 '/var/spool/cwl': {
 715                     'writable': True,
 716                     'kind': 'collection'
 717                 },
 718                 'stdout': {
 719                     'path': '/var/spool/cwl/cwl.output.json',
 720                     'kind': 'file'
 721                 },
 722                 '/var/lib/cwl/workflow': {
 723                     'portable_data_hash': '99999999999999999999999999999994+99',
 724                     'kind': 'collection'
 725                 },
 726                 '/var/lib/cwl/cwl.input.json': {
 727                     'content': {
 728                         'x': 'XxX'
 729                     },
 730                     'kind': 'json'
 731                 }
 732             }, 'state': 'Committed',
 733             'owner_uuid': None,
 734             'output_path': '/var/spool/cwl',
 735             'name': 'expect_arvworkflow.cwl#main',
 736             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 737             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 738                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 739                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 740             'cwd': '/var/spool/cwl',
 741             'runtime_constraints': {
 742                 'API': True,
 743                 'vcpus': 1,
 744                 'ram': 1073741824
 745             },
 746             'use_existing': True,
 747             'properties': {}
 748         }
 749
 750         stubs.api.container_requests().create.assert_called_with(
 751             body=JsonDiffMatcher(expect_container))
 752         self.assertEqual(capture_stdout.getvalue(),
 753                          stubs.expect_container_request_uuid + '\n')
 754
 755
 756     @mock.patch("arvados.collection.CollectionReader")
 757     @mock.patch("time.sleep")
 758     @stubs
 759     def test_submit_jobs_keepref(self, stubs, tm, reader):
 760         capture_stdout = cStringIO.StringIO()
 761
 762         with open("tests/wf/expect_arvworkflow.cwl") as f:
 763             reader().open().__enter__().read.return_value = f.read()
 764
 765         exited = arvados_cwl.main(
 766             ["--submit", "--no-wait", "--api=jobs", "--debug",
 767              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 768             capture_stdout, sys.stderr, api_client=stubs.api)
 769         self.assertEqual(exited, 0)
 770
 771         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 772         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 773         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 774         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 775         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 776         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 777         stubs.api.pipeline_instances().create.assert_called_with(
 778             body=JsonDiffMatcher(expect_pipeline))
 779
 780     @mock.patch("time.sleep")
 781     @stubs
 782     def test_submit_arvworkflow(self, stubs, tm):
 783         capture_stdout = cStringIO.StringIO()
 784
 785         with open("tests/wf/expect_arvworkflow.cwl") as f:
 786             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 787
 788         exited = arvados_cwl.main(
 789             ["--submit", "--no-wait", "--api=containers", "--debug",
 790              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 791             capture_stdout, sys.stderr, api_client=stubs.api)
 792         self.assertEqual(exited, 0)
 793
 794         expect_container = {
 795             'priority': 1,
 796             'mounts': {
 797                 '/var/spool/cwl': {
 798                     'writable': True,
 799                     'kind': 'collection'
 800                 },
 801                 'stdout': {
 802                     'path': '/var/spool/cwl/cwl.output.json',
 803                     'kind': 'file'
 804                 },
 805                 '/var/lib/cwl/workflow.json': {
 806                     'kind': 'json',
 807                     'content': {
 808                         'cwlVersion': 'v1.0',
 809                         '$graph': [
 810                             {
 811                                 'id': '#main',
 812                                 'inputs': [
 813                                     {'type': 'string', 'id': '#main/x'}
 814                                 ],
 815                                 'steps': [
 816                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 817                                      'run': '#submit_tool.cwl',
 818                                      'id': '#main/step1',
 819                                      'out': []}
 820                                 ],
 821                                 'class': 'Workflow',
 822                                 'outputs': []
 823                             },
 824                             {
 825                                 'inputs': [
 826                                     {
 827                                         'inputBinding': {'position': 1},
 828                                         'type': 'string',
 829                                         'id': '#submit_tool.cwl/x'}
 830                                 ],
 831                                 'requirements': [
 832                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 833                                 ],
 834                                 'id': '#submit_tool.cwl',
 835                                 'outputs': [],
 836                                 'baseCommand': 'cat',
 837                                 'class': 'CommandLineTool'
 838                             }
 839                         ]
 840                     }
 841                 },
 842                 '/var/lib/cwl/cwl.input.json': {
 843                     'content': {
 844                         'x': 'XxX'
 845                     },
 846                     'kind': 'json'
 847                 }
 848             }, 'state': 'Committed',
 849             'owner_uuid': None,
 850             'output_path': '/var/spool/cwl',
 851             'name': 'a test workflow',
 852             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 853             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 854                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 855                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 856             'cwd': '/var/spool/cwl',
 857             'runtime_constraints': {
 858                 'API': True,
 859                 'vcpus': 1,
 860                 'ram': 1073741824
 861             },
 862             'use_existing': True,
 863             'properties': {
 864                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 865             }
 866         }
 867
 868         stubs.api.container_requests().create.assert_called_with(
 869             body=JsonDiffMatcher(expect_container))
 870         self.assertEqual(capture_stdout.getvalue(),
 871                          stubs.expect_container_request_uuid + '\n')
 872
 873
 874     @stubs
 875     def test_submit_container_name(self, stubs):
 876         capture_stdout = cStringIO.StringIO()
 877         try:
 878             exited = arvados_cwl.main(
 879                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 880                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 881                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 882             self.assertEqual(exited, 0)
 883         except:
 884             logging.exception("")
 885
 886         expect_container = copy.deepcopy(stubs.expect_container_spec)
 887         expect_container["name"] = "hello container 123"
 888
 889         stubs.api.container_requests().create.assert_called_with(
 890             body=JsonDiffMatcher(expect_container))
 891         self.assertEqual(capture_stdout.getvalue(),
 892                          stubs.expect_container_request_uuid + '\n')
 893
 894
 895     @stubs
 896     def test_submit_container_project(self, stubs):
 897         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 898         capture_stdout = cStringIO.StringIO()
 899         try:
 900             exited = arvados_cwl.main(
 901                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 902                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904             self.assertEqual(exited, 0)
 905         except:
 906             logging.exception("")
 907
 908         expect_container = copy.deepcopy(stubs.expect_container_spec)
 909         expect_container["owner_uuid"] = project_uuid
 910         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 911                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid, '--eval-timeout=20',
 912                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 913
 914         stubs.api.container_requests().create.assert_called_with(
 915             body=JsonDiffMatcher(expect_container))
 916         self.assertEqual(capture_stdout.getvalue(),
 917                          stubs.expect_container_request_uuid + '\n')
 918
 919     @stubs
 920     def test_submit_container_eval_timeout(self, stubs):
 921         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 922         capture_stdout = cStringIO.StringIO()
 923         try:
 924             exited = arvados_cwl.main(
 925                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 926                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 927                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 928             self.assertEqual(exited, 0)
 929         except:
 930             logging.exception("")
 931
 932         expect_container = copy.deepcopy(stubs.expect_container_spec)
 933         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 934                                        '--enable-reuse', '--on-error=continue', '--eval-timeout=60.0',
 935                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 936
 937         stubs.api.container_requests().create.assert_called_with(
 938             body=JsonDiffMatcher(expect_container))
 939         self.assertEqual(capture_stdout.getvalue(),
 940                          stubs.expect_container_request_uuid + '\n')
 941
 942
 943     @stubs
 944     def test_submit_job_runner_image(self, stubs):
 945         capture_stdout = cStringIO.StringIO()
 946         try:
 947             exited = arvados_cwl.main(
 948                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 949                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 950                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 951             self.assertEqual(exited, 0)
 952         except:
 953             logging.exception("")
 954
 955         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 956
 957         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 958         stubs.api.pipeline_instances().create.assert_called_with(
 959             body=JsonDiffMatcher(expect_pipeline))
 960         self.assertEqual(capture_stdout.getvalue(),
 961                          stubs.expect_pipeline_uuid + '\n')
 962
 963     @stubs
 964     def test_submit_container_runner_image(self, stubs):
 965         capture_stdout = cStringIO.StringIO()
 966         try:
 967             exited = arvados_cwl.main(
 968                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 969                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 970                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 971             self.assertEqual(exited, 0)
 972         except:
 973             logging.exception("")
 974
 975         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 976
 977         expect_container = copy.deepcopy(stubs.expect_container_spec)
 978         stubs.api.container_requests().create.assert_called_with(
 979             body=JsonDiffMatcher(expect_container))
 980         self.assertEqual(capture_stdout.getvalue(),
 981                          stubs.expect_container_request_uuid + '\n')
 982
 983
 984     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 985     @mock.patch("cwltool.docker.get_image")
 986     @mock.patch("arvados.api")
 987     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 988         arvrunner = mock.MagicMock()
 989         arvrunner.project_uuid = ""
 990         api.return_value = mock.MagicMock()
 991         arvrunner.api = api.return_value
 992         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 993                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 994                                                                         "link_class": "docker_image_repo+tag",
 995                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 996                                                                         "owner_uuid": "",
 997                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 998                                                             {"items": [{"created_at": "",
 999                                                                         "head_uuid": "",
1000                                                                         "link_class": "docker_image_hash",
1001                                                                         "name": "123456",
1002                                                                         "owner_uuid": "",
1003                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1004         )
1005         find_one_image_hash.return_value = "123456"
1006
1007         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1008                                                                               "owner_uuid": "",
1009                                                                               "manifest_text": "",
1010                                                                               "properties": ""
1011                                                                           }], "items_available": 1, "offset": 0},)
1012         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1013         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1014                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1015
1016 class TestCreateTemplate(unittest.TestCase):
1017     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1018
1019     def _adjust_script_params(self, expect_component):
1020         expect_component['script_parameters']['x'] = {
1021             'dataclass': 'File',
1022             'required': True,
1023             'type': 'File',
1024             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1025         }
1026         expect_component['script_parameters']['y'] = {
1027             'dataclass': 'Collection',
1028             'required': True,
1029             'type': 'Directory',
1030             'value': '99999999999999999999999999999998+99',
1031         }
1032         expect_component['script_parameters']['z'] = {
1033             'dataclass': 'Collection',
1034             'required': True,
1035             'type': 'Directory',
1036         }
1037
1038     @stubs
1039     def test_create(self, stubs):
1040         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1041
1042         capture_stdout = cStringIO.StringIO()
1043
1044         exited = arvados_cwl.main(
1045             ["--create-workflow", "--debug",
1046              "--api=jobs",
1047              "--project-uuid", project_uuid,
1048              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1049             capture_stdout, sys.stderr, api_client=stubs.api)
1050         self.assertEqual(exited, 0)
1051
1052         stubs.api.pipeline_instances().create.refute_called()
1053         stubs.api.jobs().create.refute_called()
1054
1055         expect_component = copy.deepcopy(stubs.expect_job_spec)
1056         self._adjust_script_params(expect_component)
1057         expect_template = {
1058             "components": {
1059                 "submit_wf.cwl": expect_component,
1060             },
1061             "name": "submit_wf.cwl",
1062             "owner_uuid": project_uuid,
1063         }
1064         stubs.api.pipeline_templates().create.assert_called_with(
1065             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1066
1067         self.assertEqual(capture_stdout.getvalue(),
1068                          stubs.expect_pipeline_template_uuid + '\n')
1069
1070
1071     @stubs
1072     def test_create_name(self, stubs):
1073         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1074
1075         capture_stdout = cStringIO.StringIO()
1076
1077         exited = arvados_cwl.main(
1078             ["--create-workflow", "--debug",
1079              "--project-uuid", project_uuid,
1080              "--api=jobs",
1081              "--name", "testing 123",
1082              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1083             capture_stdout, sys.stderr, api_client=stubs.api)
1084         self.assertEqual(exited, 0)
1085
1086         stubs.api.pipeline_instances().create.refute_called()
1087         stubs.api.jobs().create.refute_called()
1088
1089         expect_component = copy.deepcopy(stubs.expect_job_spec)
1090         self._adjust_script_params(expect_component)
1091         expect_template = {
1092             "components": {
1093                 "testing 123": expect_component,
1094             },
1095             "name": "testing 123",
1096             "owner_uuid": project_uuid,
1097         }
1098         stubs.api.pipeline_templates().create.assert_called_with(
1099             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1100
1101         self.assertEqual(capture_stdout.getvalue(),
1102                          stubs.expect_pipeline_template_uuid + '\n')
1103
1104
1105     @stubs
1106     def test_update_name(self, stubs):
1107         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1108
1109         capture_stdout = cStringIO.StringIO()
1110
1111         exited = arvados_cwl.main(
1112             ["--update-workflow", self.existing_template_uuid,
1113              "--debug",
1114              "--project-uuid", project_uuid,
1115              "--api=jobs",
1116              "--name", "testing 123",
1117              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1118             capture_stdout, sys.stderr, api_client=stubs.api)
1119         self.assertEqual(exited, 0)
1120
1121         stubs.api.pipeline_instances().create.refute_called()
1122         stubs.api.jobs().create.refute_called()
1123
1124         expect_component = copy.deepcopy(stubs.expect_job_spec)
1125         self._adjust_script_params(expect_component)
1126         expect_template = {
1127             "components": {
1128                 "testing 123": expect_component,
1129             },
1130             "name": "testing 123",
1131             "owner_uuid": project_uuid,
1132         }
1133         stubs.api.pipeline_templates().create.refute_called()
1134         stubs.api.pipeline_templates().update.assert_called_with(
1135             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1136
1137         self.assertEqual(capture_stdout.getvalue(),
1138                          self.existing_template_uuid + '\n')
1139
1140
1141 class TestCreateWorkflow(unittest.TestCase):
1142     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1143     expect_workflow = StripYAMLComments(
1144         open("tests/wf/expect_packed.cwl").read())
1145
1146     @stubs
1147     def test_create(self, stubs):
1148         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1149
1150         capture_stdout = cStringIO.StringIO()
1151
1152         exited = arvados_cwl.main(
1153             ["--create-workflow", "--debug",
1154              "--api=containers",
1155              "--project-uuid", project_uuid,
1156              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1157             capture_stdout, sys.stderr, api_client=stubs.api)
1158         self.assertEqual(exited, 0)
1159
1160         stubs.api.pipeline_templates().create.refute_called()
1161         stubs.api.container_requests().create.refute_called()
1162
1163         body = {
1164             "workflow": {
1165                 "owner_uuid": project_uuid,
1166                 "name": "submit_wf.cwl",
1167                 "description": "",
1168                 "definition": self.expect_workflow,
1169             }
1170         }
1171         stubs.api.workflows().create.assert_called_with(
1172             body=JsonDiffMatcher(body))
1173
1174         self.assertEqual(capture_stdout.getvalue(),
1175                          stubs.expect_workflow_uuid + '\n')
1176
1177
1178     @stubs
1179     def test_create_name(self, stubs):
1180         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1181
1182         capture_stdout = cStringIO.StringIO()
1183
1184         exited = arvados_cwl.main(
1185             ["--create-workflow", "--debug",
1186              "--api=containers",
1187              "--project-uuid", project_uuid,
1188              "--name", "testing 123",
1189              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1190             capture_stdout, sys.stderr, api_client=stubs.api)
1191         self.assertEqual(exited, 0)
1192
1193         stubs.api.pipeline_templates().create.refute_called()
1194         stubs.api.container_requests().create.refute_called()
1195
1196         body = {
1197             "workflow": {
1198                 "owner_uuid": project_uuid,
1199                 "name": "testing 123",
1200                 "description": "",
1201                 "definition": self.expect_workflow,
1202             }
1203         }
1204         stubs.api.workflows().create.assert_called_with(
1205             body=JsonDiffMatcher(body))
1206
1207         self.assertEqual(capture_stdout.getvalue(),
1208                          stubs.expect_workflow_uuid + '\n')
1209
1210     @stubs
1211     def test_incompatible_api(self, stubs):
1212         capture_stderr = cStringIO.StringIO()
1213         logging.getLogger('arvados.cwl-runner').addHandler(
1214             logging.StreamHandler(capture_stderr))
1215
1216         exited = arvados_cwl.main(
1217             ["--update-workflow", self.existing_workflow_uuid,
1218              "--api=jobs",
1219              "--debug",
1220              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1221             sys.stderr, sys.stderr, api_client=stubs.api)
1222         self.assertEqual(exited, 1)
1223         self.assertRegexpMatches(
1224             capture_stderr.getvalue(),
1225             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1226
1227     @stubs
1228     def test_update(self, stubs):
1229         capture_stdout = cStringIO.StringIO()
1230
1231         exited = arvados_cwl.main(
1232             ["--update-workflow", self.existing_workflow_uuid,
1233              "--debug",
1234              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1235             capture_stdout, sys.stderr, api_client=stubs.api)
1236         self.assertEqual(exited, 0)
1237
1238         body = {
1239             "workflow": {
1240                 "name": "submit_wf.cwl",
1241                 "description": "",
1242                 "definition": self.expect_workflow,
1243             }
1244         }
1245         stubs.api.workflows().update.assert_called_with(
1246             uuid=self.existing_workflow_uuid,
1247             body=JsonDiffMatcher(body))
1248         self.assertEqual(capture_stdout.getvalue(),
1249                          self.existing_workflow_uuid + '\n')
1250
1251
1252     @stubs
1253     def test_update_name(self, stubs):
1254         capture_stdout = cStringIO.StringIO()
1255
1256         exited = arvados_cwl.main(
1257             ["--update-workflow", self.existing_workflow_uuid,
1258              "--debug", "--name", "testing 123",
1259              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1260             capture_stdout, sys.stderr, api_client=stubs.api)
1261         self.assertEqual(exited, 0)
1262
1263         body = {
1264             "workflow": {
1265                 "name": "testing 123",
1266                 "description": "",
1267                 "definition": self.expect_workflow,
1268             }
1269         }
1270         stubs.api.workflows().update.assert_called_with(
1271             uuid=self.existing_workflow_uuid,
1272             body=JsonDiffMatcher(body))
1273         self.assertEqual(capture_stdout.getvalue(),
1274                          self.existing_workflow_uuid + '\n')
1275
1276
1277     @stubs
1278     def test_create_collection_per_tool(self, stubs):
1279         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1280
1281         capture_stdout = cStringIO.StringIO()
1282
1283         exited = arvados_cwl.main(
1284             ["--create-workflow", "--debug",
1285              "--api=containers",
1286              "--project-uuid", project_uuid,
1287              "tests/collection_per_tool/collection_per_tool.cwl"],
1288             capture_stdout, sys.stderr, api_client=stubs.api)
1289         self.assertEqual(exited, 0)
1290
1291         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1292         expect_workflow = StripYAMLComments(open(toolfile).read())
1293
1294         body = {
1295             "workflow": {
1296                 "owner_uuid": project_uuid,
1297                 "name": "collection_per_tool.cwl",
1298                 "description": "",
1299                 "definition": expect_workflow,
1300             }
1301         }
1302         stubs.api.workflows().create.assert_called_with(
1303             body=JsonDiffMatcher(body))
1304
1305         self.assertEqual(capture_stdout.getvalue(),
1306                          stubs.expect_workflow_uuid + '\n')
1307
1308 class TestTemplateInputs(unittest.TestCase):
1309     expect_template = {
1310         "components": {
1311             "inputs_test.cwl": {
1312                 'runtime_constraints': {
1313                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1314                     'min_ram_mb_per_node': 1024
1315                 },
1316                 'script_parameters': {
1317                     'cwl:tool':
1318                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1319                     'optionalFloatInput': None,
1320                     'fileInput': {
1321                         'type': 'File',
1322                         'dataclass': 'File',
1323                         'required': True,
1324                         'title': "It's a file; we expect to find some characters in it.",
1325                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1326                     },
1327                     'floatInput': {
1328                         'type': 'float',
1329                         'dataclass': 'number',
1330                         'required': True,
1331                         'title': 'Floats like a duck',
1332                         'default': 0.1,
1333                         'value': 0.1,
1334                     },
1335                     'optionalFloatInput': {
1336                         'type': ['null', 'float'],
1337                         'dataclass': 'number',
1338                         'required': False,
1339                     },
1340                     'boolInput': {
1341                         'type': 'boolean',
1342                         'dataclass': 'boolean',
1343                         'required': True,
1344                         'title': 'True or false?',
1345                     },
1346                 },
1347                 'repository': 'arvados',
1348                 'script_version': 'master',
1349                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1350                 'script': 'cwl-runner',
1351             },
1352         },
1353         "name": "inputs_test.cwl",
1354     }
1355
1356     @stubs
1357     def test_inputs_empty(self, stubs):
1358         exited = arvados_cwl.main(
1359             ["--create-template",
1360              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1361             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1362         self.assertEqual(exited, 0)
1363
1364         stubs.api.pipeline_templates().create.assert_called_with(
1365             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1366
1367     @stubs
1368     def test_inputs(self, stubs):
1369         exited = arvados_cwl.main(
1370             ["--create-template",
1371              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1372             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1373         self.assertEqual(exited, 0)
1374
1375         expect_template = copy.deepcopy(self.expect_template)
1376         params = expect_template[
1377             "components"]["inputs_test.cwl"]["script_parameters"]
1378         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1379         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1380         params["floatInput"]["value"] = 1.234
1381         params["boolInput"]["value"] = True
1382
1383         stubs.api.pipeline_templates().create.assert_called_with(
1384             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)