sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 1,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'state': 'Committed',
 235             'owner_uuid': None,
 236             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 237                         '--enable-reuse', '--on-error=continue',
 238                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 239             'name': 'submit_wf.cwl',
 240             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 241             'output_path': '/var/spool/cwl',
 242             'cwd': '/var/spool/cwl',
 243             'runtime_constraints': {
 244                 'API': True,
 245                 'vcpus': 1,
 246                 'ram': 1024*1024*1024
 247             },
 248             'use_existing': True,
 249             'properties': {}
 250         }
 251
 252         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 253         stubs.api.workflows().create().execute.return_value = {
 254             "uuid": stubs.expect_workflow_uuid,
 255         }
 256         def update_mock(**kwargs):
 257             stubs.updated_uuid = kwargs.get('uuid')
 258             return mock.DEFAULT
 259         stubs.api.workflows().update.side_effect = update_mock
 260         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 261             "uuid": stubs.updated_uuid,
 262         }
 263
 264         return func(self, stubs, *args, **kwargs)
 265     return wrapped
 266
 267
 268 class TestSubmit(unittest.TestCase):
 269     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit(self, stubs, tm, arvdock):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--api=jobs", "--debug",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.api.collections().create.assert_has_calls([
 281             mock.call(body=JsonDiffMatcher({
 282                 'manifest_text':
 283                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 284                 'replication_desired': None,
 285                 'name': 'submit_tool.cwl dependencies',
 286             }), ensure_unique_name=True),
 287             mock.call(body=JsonDiffMatcher({
 288                 'manifest_text':
 289                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 290                 'replication_desired': None,
 291                 'name': 'submit_wf.cwl input',
 292             }), ensure_unique_name=True),
 293             mock.call(body=JsonDiffMatcher({
 294                 'manifest_text':
 295                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 296                 'replication_desired': None,
 297                 'name': 'submit_wf.cwl',
 298             }), ensure_unique_name=True)        ])
 299
 300         arvdock.assert_has_calls([
 301             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 302             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 303         ])
 304
 305         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 306         stubs.api.pipeline_instances().create.assert_called_with(
 307             body=JsonDiffMatcher(expect_pipeline))
 308         self.assertEqual(capture_stdout.getvalue(),
 309                          stubs.expect_pipeline_uuid + '\n')
 310
 311
 312     @mock.patch("time.sleep")
 313     @stubs
 314     def test_submit_no_reuse(self, stubs, tm):
 315         capture_stdout = cStringIO.StringIO()
 316         exited = arvados_cwl.main(
 317             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 318              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 319             capture_stdout, sys.stderr, api_client=stubs.api)
 320         self.assertEqual(exited, 0)
 321
 322         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 323         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 324         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 325
 326         stubs.api.pipeline_instances().create.assert_called_with(
 327             body=JsonDiffMatcher(expect_pipeline))
 328         self.assertEqual(capture_stdout.getvalue(),
 329                          stubs.expect_pipeline_uuid + '\n')
 330
 331     @mock.patch("time.sleep")
 332     @stubs
 333     def test_submit_on_error(self, stubs, tm):
 334         capture_stdout = cStringIO.StringIO()
 335         exited = arvados_cwl.main(
 336             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 337              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 338             capture_stdout, sys.stderr, api_client=stubs.api)
 339         self.assertEqual(exited, 0)
 340
 341         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 342         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 343
 344         stubs.api.pipeline_instances().create.assert_called_with(
 345             body=JsonDiffMatcher(expect_pipeline))
 346         self.assertEqual(capture_stdout.getvalue(),
 347                          stubs.expect_pipeline_uuid + '\n')
 348
 349
 350     @mock.patch("time.sleep")
 351     @stubs
 352     def test_submit_runner_ram(self, stubs, tm):
 353         capture_stdout = cStringIO.StringIO()
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             capture_stdout, sys.stderr, api_client=stubs.api)
 358         self.assertEqual(exited, 0)
 359
 360         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 361         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 362
 363         stubs.api.pipeline_instances().create.assert_called_with(
 364             body=JsonDiffMatcher(expect_pipeline))
 365         self.assertEqual(capture_stdout.getvalue(),
 366                          stubs.expect_pipeline_uuid + '\n')
 367
 368
 369     @mock.patch("time.sleep")
 370     @stubs
 371     def test_submit_invalid_runner_ram(self, stubs, tm):
 372         capture_stdout = cStringIO.StringIO()
 373         exited = arvados_cwl.main(
 374             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 375              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 376             capture_stdout, sys.stderr, api_client=stubs.api)
 377         self.assertEqual(exited, 1)
 378
 379     @mock.patch("time.sleep")
 380     @stubs
 381     def test_submit_output_name(self, stubs, tm):
 382         output_name = "test_output_name"
 383
 384         capture_stdout = cStringIO.StringIO()
 385         exited = arvados_cwl.main(
 386             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 388             capture_stdout, sys.stderr, api_client=stubs.api)
 389         self.assertEqual(exited, 0)
 390
 391         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 392         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 393
 394         stubs.api.pipeline_instances().create.assert_called_with(
 395             body=JsonDiffMatcher(expect_pipeline))
 396         self.assertEqual(capture_stdout.getvalue(),
 397                          stubs.expect_pipeline_uuid + '\n')
 398
 399
 400     @mock.patch("time.sleep")
 401     @stubs
 402     def test_submit_pipeline_name(self, stubs, tm):
 403         capture_stdout = cStringIO.StringIO()
 404         exited = arvados_cwl.main(
 405             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 406              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 407             capture_stdout, sys.stderr, api_client=stubs.api)
 408         self.assertEqual(exited, 0)
 409
 410         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 411         expect_pipeline["name"] = "hello job 123"
 412
 413         stubs.api.pipeline_instances().create.assert_called_with(
 414             body=JsonDiffMatcher(expect_pipeline))
 415         self.assertEqual(capture_stdout.getvalue(),
 416                          stubs.expect_pipeline_uuid + '\n')
 417
 418     @mock.patch("time.sleep")
 419     @stubs
 420     def test_submit_output_tags(self, stubs, tm):
 421         output_tags = "tag0,tag1,tag2"
 422
 423         capture_stdout = cStringIO.StringIO()
 424         exited = arvados_cwl.main(
 425             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 426              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 427             capture_stdout, sys.stderr, api_client=stubs.api)
 428         self.assertEqual(exited, 0)
 429
 430         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 431         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 432
 433         stubs.api.pipeline_instances().create.assert_called_with(
 434             body=JsonDiffMatcher(expect_pipeline))
 435         self.assertEqual(capture_stdout.getvalue(),
 436                          stubs.expect_pipeline_uuid + '\n')
 437
 438     @mock.patch("time.sleep")
 439     @stubs
 440     def test_submit_with_project_uuid(self, stubs, tm):
 441         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 442
 443         exited = arvados_cwl.main(
 444             ["--submit", "--no-wait",
 445              "--project-uuid", project_uuid,
 446              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 447             sys.stdout, sys.stderr, api_client=stubs.api)
 448         self.assertEqual(exited, 0)
 449
 450         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 451         expect_pipeline["owner_uuid"] = project_uuid
 452         stubs.api.pipeline_instances().create.assert_called_with(
 453             body=JsonDiffMatcher(expect_pipeline))
 454
 455     @stubs
 456     def test_submit_container(self, stubs):
 457         capture_stdout = cStringIO.StringIO()
 458         try:
 459             exited = arvados_cwl.main(
 460                 ["--submit", "--no-wait", "--api=containers", "--debug",
 461                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 462                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 463             self.assertEqual(exited, 0)
 464         except:
 465             logging.exception("")
 466
 467         stubs.api.collections().create.assert_has_calls([
 468             mock.call(body=JsonDiffMatcher({
 469                 'manifest_text':
 470                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 471                 'replication_desired': None,
 472                 'name': 'submit_tool.cwl dependencies',
 473             }), ensure_unique_name=True),
 474             mock.call(body=JsonDiffMatcher({
 475                 'manifest_text':
 476                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 477                 'replication_desired': None,
 478                 'name': 'submit_wf.cwl input',
 479             }), ensure_unique_name=True)])
 480
 481         expect_container = copy.deepcopy(stubs.expect_container_spec)
 482         stubs.api.container_requests().create.assert_called_with(
 483             body=JsonDiffMatcher(expect_container))
 484         self.assertEqual(capture_stdout.getvalue(),
 485                          stubs.expect_container_request_uuid + '\n')
 486
 487     @stubs
 488     def test_submit_container_no_reuse(self, stubs):
 489         capture_stdout = cStringIO.StringIO()
 490         try:
 491             exited = arvados_cwl.main(
 492                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 493                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 494                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 495             self.assertEqual(exited, 0)
 496         except:
 497             logging.exception("")
 498
 499         expect_container = copy.deepcopy(stubs.expect_container_spec)
 500         expect_container["command"] = [
 501             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 502             '--disable-reuse', '--on-error=continue',
 503             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 504         expect_container["use_existing"] = False
 505
 506         stubs.api.container_requests().create.assert_called_with(
 507             body=JsonDiffMatcher(expect_container))
 508         self.assertEqual(capture_stdout.getvalue(),
 509                          stubs.expect_container_request_uuid + '\n')
 510
 511
 512     @stubs
 513     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 514         capture_stdout = cStringIO.StringIO()
 515
 516         exited = arvados_cwl.main(
 517             ["--submit", "--no-wait", "--api=containers", "--debug",
 518              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 519             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 520         self.assertEqual(exited, 0)
 521
 522         expect_container = copy.deepcopy(stubs.expect_container_spec)
 523         expect_container["command"] = [
 524             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 525             '--disable-reuse', '--on-error=continue',
 526             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 527         expect_container["use_existing"] = False
 528         expect_container["name"] = "submit_wf_no_reuse.cwl"
 529         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 530             {
 531                 "class": "http://arvados.org/cwl#ReuseRequirement",
 532                 "enableReuse": False,
 533             },
 534         ]
 535         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 536             "arv": "http://arvados.org/cwl#",
 537             "cwltool": "http://commonwl.org/cwltool#"
 538         }
 539
 540         stubs.api.container_requests().create.assert_called_with(
 541             body=JsonDiffMatcher(expect_container))
 542         self.assertEqual(capture_stdout.getvalue(),
 543                          stubs.expect_container_request_uuid + '\n')
 544
 545
 546     @stubs
 547     def test_submit_container_on_error(self, stubs):
 548         capture_stdout = cStringIO.StringIO()
 549         try:
 550             exited = arvados_cwl.main(
 551                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 552                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 553                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 554             self.assertEqual(exited, 0)
 555         except:
 556             logging.exception("")
 557
 558         expect_container = copy.deepcopy(stubs.expect_container_spec)
 559         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 560                                                   '--enable-reuse', '--on-error=stop',
 561                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 562
 563         stubs.api.container_requests().create.assert_called_with(
 564             body=JsonDiffMatcher(expect_container))
 565         self.assertEqual(capture_stdout.getvalue(),
 566                          stubs.expect_container_request_uuid + '\n')
 567
 568     @stubs
 569     def test_submit_container_output_name(self, stubs):
 570         output_name = "test_output_name"
 571
 572         capture_stdout = cStringIO.StringIO()
 573         try:
 574             exited = arvados_cwl.main(
 575                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 576                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 577                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 578             self.assertEqual(exited, 0)
 579         except:
 580             logging.exception("")
 581
 582         expect_container = copy.deepcopy(stubs.expect_container_spec)
 583         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 584                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
 585                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 586         expect_container["output_name"] = output_name
 587
 588         stubs.api.container_requests().create.assert_called_with(
 589             body=JsonDiffMatcher(expect_container))
 590         self.assertEqual(capture_stdout.getvalue(),
 591                          stubs.expect_container_request_uuid + '\n')
 592
 593
 594     @stubs
 595     def test_submit_container_output_ttl(self, stubs):
 596         capture_stdout = cStringIO.StringIO()
 597         try:
 598             exited = arvados_cwl.main(
 599                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 600                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 601                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 602             self.assertEqual(exited, 0)
 603         except:
 604             logging.exception("")
 605
 606         expect_container = copy.deepcopy(stubs.expect_container_spec)
 607         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 608                                        '--enable-reuse', '--on-error=continue',
 609                                        "--intermediate-output-ttl=3600",
 610                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 611
 612         stubs.api.container_requests().create.assert_called_with(
 613             body=JsonDiffMatcher(expect_container))
 614         self.assertEqual(capture_stdout.getvalue(),
 615                          stubs.expect_container_request_uuid + '\n')
 616
 617     @stubs
 618     def test_submit_container_trash_intermediate(self, stubs):
 619         capture_stdout = cStringIO.StringIO()
 620         try:
 621             exited = arvados_cwl.main(
 622                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 623                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 624                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 625             self.assertEqual(exited, 0)
 626         except:
 627             logging.exception("")
 628
 629         expect_container = copy.deepcopy(stubs.expect_container_spec)
 630         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 631                                        '--enable-reuse', '--on-error=continue',
 632                                        "--trash-intermediate",
 633                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 634
 635         stubs.api.container_requests().create.assert_called_with(
 636             body=JsonDiffMatcher(expect_container))
 637         self.assertEqual(capture_stdout.getvalue(),
 638                          stubs.expect_container_request_uuid + '\n')
 639
 640     @stubs
 641     def test_submit_container_output_tags(self, stubs):
 642         output_tags = "tag0,tag1,tag2"
 643
 644         capture_stdout = cStringIO.StringIO()
 645         try:
 646             exited = arvados_cwl.main(
 647                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 648                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 649                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 650             self.assertEqual(exited, 0)
 651         except:
 652             logging.exception("")
 653
 654         expect_container = copy.deepcopy(stubs.expect_container_spec)
 655         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 656                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
 657                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 658
 659         stubs.api.container_requests().create.assert_called_with(
 660             body=JsonDiffMatcher(expect_container))
 661         self.assertEqual(capture_stdout.getvalue(),
 662                          stubs.expect_container_request_uuid + '\n')
 663
 664     @stubs
 665     def test_submit_container_runner_ram(self, stubs):
 666         capture_stdout = cStringIO.StringIO()
 667         try:
 668             exited = arvados_cwl.main(
 669                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 670                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 671                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 672             self.assertEqual(exited, 0)
 673         except:
 674             logging.exception("")
 675
 676         expect_container = copy.deepcopy(stubs.expect_container_spec)
 677         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 678
 679         stubs.api.container_requests().create.assert_called_with(
 680             body=JsonDiffMatcher(expect_container))
 681         self.assertEqual(capture_stdout.getvalue(),
 682                          stubs.expect_container_request_uuid + '\n')
 683
 684     @mock.patch("arvados.collection.CollectionReader")
 685     @mock.patch("time.sleep")
 686     @stubs
 687     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 688         capture_stdout = cStringIO.StringIO()
 689         exited = arvados_cwl.main(
 690             ["--submit", "--no-wait", "--api=containers", "--debug",
 691              "tests/wf/submit_keepref_wf.cwl"],
 692             capture_stdout, sys.stderr, api_client=stubs.api)
 693         self.assertEqual(exited, 0)
 694
 695
 696     @mock.patch("arvados.collection.CollectionReader")
 697     @mock.patch("time.sleep")
 698     @stubs
 699     def test_submit_keepref(self, stubs, tm, reader):
 700         capture_stdout = cStringIO.StringIO()
 701
 702         with open("tests/wf/expect_arvworkflow.cwl") as f:
 703             reader().open().__enter__().read.return_value = f.read()
 704
 705         exited = arvados_cwl.main(
 706             ["--submit", "--no-wait", "--api=containers", "--debug",
 707              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 708             capture_stdout, sys.stderr, api_client=stubs.api)
 709         self.assertEqual(exited, 0)
 710
 711         expect_container = {
 712             'priority': 1,
 713             'mounts': {
 714                 '/var/spool/cwl': {
 715                     'writable': True,
 716                     'kind': 'collection'
 717                 },
 718                 'stdout': {
 719                     'path': '/var/spool/cwl/cwl.output.json',
 720                     'kind': 'file'
 721                 },
 722                 '/var/lib/cwl/workflow': {
 723                     'portable_data_hash': '99999999999999999999999999999994+99',
 724                     'kind': 'collection'
 725                 },
 726                 '/var/lib/cwl/cwl.input.json': {
 727                     'content': {
 728                         'x': 'XxX'
 729                     },
 730                     'kind': 'json'
 731                 }
 732             }, 'state': 'Committed',
 733             'owner_uuid': None,
 734             'output_path': '/var/spool/cwl',
 735             'name': 'expect_arvworkflow.cwl#main',
 736             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 737             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 738                         '--enable-reuse', '--on-error=continue',
 739                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 740             'cwd': '/var/spool/cwl',
 741             'runtime_constraints': {
 742                 'API': True,
 743                 'vcpus': 1,
 744                 'ram': 1073741824
 745             },
 746             'use_existing': True,
 747             'properties': {}
 748         }
 749
 750         stubs.api.container_requests().create.assert_called_with(
 751             body=JsonDiffMatcher(expect_container))
 752         self.assertEqual(capture_stdout.getvalue(),
 753                          stubs.expect_container_request_uuid + '\n')
 754
 755
 756     @mock.patch("arvados.collection.CollectionReader")
 757     @mock.patch("time.sleep")
 758     @stubs
 759     def test_submit_jobs_keepref(self, stubs, tm, reader):
 760         capture_stdout = cStringIO.StringIO()
 761
 762         with open("tests/wf/expect_arvworkflow.cwl") as f:
 763             reader().open().__enter__().read.return_value = f.read()
 764
 765         exited = arvados_cwl.main(
 766             ["--submit", "--no-wait", "--api=jobs", "--debug",
 767              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 768             capture_stdout, sys.stderr, api_client=stubs.api)
 769         self.assertEqual(exited, 0)
 770
 771         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 772         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 773         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 774         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 775         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 776         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 777         stubs.api.pipeline_instances().create.assert_called_with(
 778             body=JsonDiffMatcher(expect_pipeline))
 779
 780     @mock.patch("time.sleep")
 781     @stubs
 782     def test_submit_arvworkflow(self, stubs, tm):
 783         capture_stdout = cStringIO.StringIO()
 784
 785         with open("tests/wf/expect_arvworkflow.cwl") as f:
 786             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 787
 788         exited = arvados_cwl.main(
 789             ["--submit", "--no-wait", "--api=containers", "--debug",
 790              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 791             capture_stdout, sys.stderr, api_client=stubs.api)
 792         self.assertEqual(exited, 0)
 793
 794         expect_container = {
 795             'priority': 1,
 796             'mounts': {
 797                 '/var/spool/cwl': {
 798                     'writable': True,
 799                     'kind': 'collection'
 800                 },
 801                 'stdout': {
 802                     'path': '/var/spool/cwl/cwl.output.json',
 803                     'kind': 'file'
 804                 },
 805                 '/var/lib/cwl/workflow.json': {
 806                     'kind': 'json',
 807                     'content': {
 808                         'cwlVersion': 'v1.0',
 809                         '$graph': [
 810                             {
 811                                 'id': '#main',
 812                                 'inputs': [
 813                                     {'type': 'string', 'id': '#main/x'}
 814                                 ],
 815                                 'steps': [
 816                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 817                                      'run': '#submit_tool.cwl',
 818                                      'id': '#main/step1',
 819                                      'out': []}
 820                                 ],
 821                                 'class': 'Workflow',
 822                                 'outputs': []
 823                             },
 824                             {
 825                                 'inputs': [
 826                                     {
 827                                         'inputBinding': {'position': 1},
 828                                         'type': 'string',
 829                                         'id': '#submit_tool.cwl/x'}
 830                                 ],
 831                                 'requirements': [
 832                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 833                                 ],
 834                                 'id': '#submit_tool.cwl',
 835                                 'outputs': [],
 836                                 'baseCommand': 'cat',
 837                                 'class': 'CommandLineTool'
 838                             }
 839                         ]
 840                     }
 841                 },
 842                 '/var/lib/cwl/cwl.input.json': {
 843                     'content': {
 844                         'x': 'XxX'
 845                     },
 846                     'kind': 'json'
 847                 }
 848             }, 'state': 'Committed',
 849             'owner_uuid': None,
 850             'output_path': '/var/spool/cwl',
 851             'name': 'a test workflow',
 852             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 853             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 854                         '--enable-reuse', '--on-error=continue',
 855                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 856             'cwd': '/var/spool/cwl',
 857             'runtime_constraints': {
 858                 'API': True,
 859                 'vcpus': 1,
 860                 'ram': 1073741824
 861             },
 862             'use_existing': True,
 863             'properties': {
 864                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 865             }
 866         }
 867
 868         stubs.api.container_requests().create.assert_called_with(
 869             body=JsonDiffMatcher(expect_container))
 870         self.assertEqual(capture_stdout.getvalue(),
 871                          stubs.expect_container_request_uuid + '\n')
 872
 873
 874     @stubs
 875     def test_submit_container_name(self, stubs):
 876         capture_stdout = cStringIO.StringIO()
 877         try:
 878             exited = arvados_cwl.main(
 879                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 880                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 881                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 882             self.assertEqual(exited, 0)
 883         except:
 884             logging.exception("")
 885
 886         expect_container = copy.deepcopy(stubs.expect_container_spec)
 887         expect_container["name"] = "hello container 123"
 888
 889         stubs.api.container_requests().create.assert_called_with(
 890             body=JsonDiffMatcher(expect_container))
 891         self.assertEqual(capture_stdout.getvalue(),
 892                          stubs.expect_container_request_uuid + '\n')
 893
 894
 895     @stubs
 896     def test_submit_container_project(self, stubs):
 897         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 898         capture_stdout = cStringIO.StringIO()
 899         try:
 900             exited = arvados_cwl.main(
 901                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 902                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 903                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 904             self.assertEqual(exited, 0)
 905         except:
 906             logging.exception("")
 907
 908         expect_container = copy.deepcopy(stubs.expect_container_spec)
 909         expect_container["owner_uuid"] = project_uuid
 910         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 911                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid,
 912                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 913
 914         stubs.api.container_requests().create.assert_called_with(
 915             body=JsonDiffMatcher(expect_container))
 916         self.assertEqual(capture_stdout.getvalue(),
 917                          stubs.expect_container_request_uuid + '\n')
 918
 919
 920     @stubs
 921     def test_submit_job_runner_image(self, stubs):
 922         capture_stdout = cStringIO.StringIO()
 923         try:
 924             exited = arvados_cwl.main(
 925                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 926                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 927                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 928             self.assertEqual(exited, 0)
 929         except:
 930             logging.exception("")
 931
 932         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 933
 934         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 935         stubs.api.pipeline_instances().create.assert_called_with(
 936             body=JsonDiffMatcher(expect_pipeline))
 937         self.assertEqual(capture_stdout.getvalue(),
 938                          stubs.expect_pipeline_uuid + '\n')
 939
 940     @stubs
 941     def test_submit_container_runner_image(self, stubs):
 942         capture_stdout = cStringIO.StringIO()
 943         try:
 944             exited = arvados_cwl.main(
 945                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 946                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 947                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 948             self.assertEqual(exited, 0)
 949         except:
 950             logging.exception("")
 951
 952         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 953
 954         expect_container = copy.deepcopy(stubs.expect_container_spec)
 955         stubs.api.container_requests().create.assert_called_with(
 956             body=JsonDiffMatcher(expect_container))
 957         self.assertEqual(capture_stdout.getvalue(),
 958                          stubs.expect_container_request_uuid + '\n')
 959
 960
 961     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 962     @mock.patch("cwltool.docker.get_image")
 963     @mock.patch("arvados.api")
 964     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 965         arvrunner = mock.MagicMock()
 966         arvrunner.project_uuid = ""
 967         api.return_value = mock.MagicMock()
 968         arvrunner.api = api.return_value
 969         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
 970                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 971                                                                         "link_class": "docker_image_repo+tag",
 972                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 973                                                                         "owner_uuid": "",
 974                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 975                                                             {"items": [{"created_at": "",
 976                                                                         "head_uuid": "",
 977                                                                         "link_class": "docker_image_hash",
 978                                                                         "name": "123456",
 979                                                                         "owner_uuid": "",
 980                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
 981         )
 982         find_one_image_hash.return_value = "123456"
 983
 984         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
 985                                                                               "owner_uuid": "",
 986                                                                               "manifest_text": "",
 987                                                                               "properties": ""
 988                                                                           }], "items_available": 1, "offset": 0},)
 989         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 990         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
 991                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 992
 993 class TestCreateTemplate(unittest.TestCase):
 994     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 995
 996     def _adjust_script_params(self, expect_component):
 997         expect_component['script_parameters']['x'] = {
 998             'dataclass': 'File',
 999             'required': True,
1000             'type': 'File',
1001             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1002         }
1003         expect_component['script_parameters']['y'] = {
1004             'dataclass': 'Collection',
1005             'required': True,
1006             'type': 'Directory',
1007             'value': '99999999999999999999999999999998+99',
1008         }
1009         expect_component['script_parameters']['z'] = {
1010             'dataclass': 'Collection',
1011             'required': True,
1012             'type': 'Directory',
1013         }
1014
1015     @stubs
1016     def test_create(self, stubs):
1017         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1018
1019         capture_stdout = cStringIO.StringIO()
1020
1021         exited = arvados_cwl.main(
1022             ["--create-workflow", "--debug",
1023              "--api=jobs",
1024              "--project-uuid", project_uuid,
1025              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1026             capture_stdout, sys.stderr, api_client=stubs.api)
1027         self.assertEqual(exited, 0)
1028
1029         stubs.api.pipeline_instances().create.refute_called()
1030         stubs.api.jobs().create.refute_called()
1031
1032         expect_component = copy.deepcopy(stubs.expect_job_spec)
1033         self._adjust_script_params(expect_component)
1034         expect_template = {
1035             "components": {
1036                 "submit_wf.cwl": expect_component,
1037             },
1038             "name": "submit_wf.cwl",
1039             "owner_uuid": project_uuid,
1040         }
1041         stubs.api.pipeline_templates().create.assert_called_with(
1042             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1043
1044         self.assertEqual(capture_stdout.getvalue(),
1045                          stubs.expect_pipeline_template_uuid + '\n')
1046
1047
1048     @stubs
1049     def test_create_name(self, stubs):
1050         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1051
1052         capture_stdout = cStringIO.StringIO()
1053
1054         exited = arvados_cwl.main(
1055             ["--create-workflow", "--debug",
1056              "--project-uuid", project_uuid,
1057              "--api=jobs",
1058              "--name", "testing 123",
1059              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1060             capture_stdout, sys.stderr, api_client=stubs.api)
1061         self.assertEqual(exited, 0)
1062
1063         stubs.api.pipeline_instances().create.refute_called()
1064         stubs.api.jobs().create.refute_called()
1065
1066         expect_component = copy.deepcopy(stubs.expect_job_spec)
1067         self._adjust_script_params(expect_component)
1068         expect_template = {
1069             "components": {
1070                 "testing 123": expect_component,
1071             },
1072             "name": "testing 123",
1073             "owner_uuid": project_uuid,
1074         }
1075         stubs.api.pipeline_templates().create.assert_called_with(
1076             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1077
1078         self.assertEqual(capture_stdout.getvalue(),
1079                          stubs.expect_pipeline_template_uuid + '\n')
1080
1081
1082     @stubs
1083     def test_update_name(self, stubs):
1084         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1085
1086         capture_stdout = cStringIO.StringIO()
1087
1088         exited = arvados_cwl.main(
1089             ["--update-workflow", self.existing_template_uuid,
1090              "--debug",
1091              "--project-uuid", project_uuid,
1092              "--api=jobs",
1093              "--name", "testing 123",
1094              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1095             capture_stdout, sys.stderr, api_client=stubs.api)
1096         self.assertEqual(exited, 0)
1097
1098         stubs.api.pipeline_instances().create.refute_called()
1099         stubs.api.jobs().create.refute_called()
1100
1101         expect_component = copy.deepcopy(stubs.expect_job_spec)
1102         self._adjust_script_params(expect_component)
1103         expect_template = {
1104             "components": {
1105                 "testing 123": expect_component,
1106             },
1107             "name": "testing 123",
1108             "owner_uuid": project_uuid,
1109         }
1110         stubs.api.pipeline_templates().create.refute_called()
1111         stubs.api.pipeline_templates().update.assert_called_with(
1112             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1113
1114         self.assertEqual(capture_stdout.getvalue(),
1115                          self.existing_template_uuid + '\n')
1116
1117
1118 class TestCreateWorkflow(unittest.TestCase):
1119     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1120     expect_workflow = StripYAMLComments(
1121         open("tests/wf/expect_packed.cwl").read())
1122
1123     @stubs
1124     def test_create(self, stubs):
1125         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1126
1127         capture_stdout = cStringIO.StringIO()
1128
1129         exited = arvados_cwl.main(
1130             ["--create-workflow", "--debug",
1131              "--api=containers",
1132              "--project-uuid", project_uuid,
1133              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1134             capture_stdout, sys.stderr, api_client=stubs.api)
1135         self.assertEqual(exited, 0)
1136
1137         stubs.api.pipeline_templates().create.refute_called()
1138         stubs.api.container_requests().create.refute_called()
1139
1140         body = {
1141             "workflow": {
1142                 "owner_uuid": project_uuid,
1143                 "name": "submit_wf.cwl",
1144                 "description": "",
1145                 "definition": self.expect_workflow,
1146             }
1147         }
1148         stubs.api.workflows().create.assert_called_with(
1149             body=JsonDiffMatcher(body))
1150
1151         self.assertEqual(capture_stdout.getvalue(),
1152                          stubs.expect_workflow_uuid + '\n')
1153
1154
1155     @stubs
1156     def test_create_name(self, stubs):
1157         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1158
1159         capture_stdout = cStringIO.StringIO()
1160
1161         exited = arvados_cwl.main(
1162             ["--create-workflow", "--debug",
1163              "--api=containers",
1164              "--project-uuid", project_uuid,
1165              "--name", "testing 123",
1166              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1167             capture_stdout, sys.stderr, api_client=stubs.api)
1168         self.assertEqual(exited, 0)
1169
1170         stubs.api.pipeline_templates().create.refute_called()
1171         stubs.api.container_requests().create.refute_called()
1172
1173         body = {
1174             "workflow": {
1175                 "owner_uuid": project_uuid,
1176                 "name": "testing 123",
1177                 "description": "",
1178                 "definition": self.expect_workflow,
1179             }
1180         }
1181         stubs.api.workflows().create.assert_called_with(
1182             body=JsonDiffMatcher(body))
1183
1184         self.assertEqual(capture_stdout.getvalue(),
1185                          stubs.expect_workflow_uuid + '\n')
1186
1187     @stubs
1188     def test_incompatible_api(self, stubs):
1189         capture_stderr = cStringIO.StringIO()
1190         logging.getLogger('arvados.cwl-runner').addHandler(
1191             logging.StreamHandler(capture_stderr))
1192
1193         exited = arvados_cwl.main(
1194             ["--update-workflow", self.existing_workflow_uuid,
1195              "--api=jobs",
1196              "--debug",
1197              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1198             sys.stderr, sys.stderr, api_client=stubs.api)
1199         self.assertEqual(exited, 1)
1200         self.assertRegexpMatches(
1201             capture_stderr.getvalue(),
1202             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1203
1204     @stubs
1205     def test_update(self, stubs):
1206         capture_stdout = cStringIO.StringIO()
1207
1208         exited = arvados_cwl.main(
1209             ["--update-workflow", self.existing_workflow_uuid,
1210              "--debug",
1211              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1212             capture_stdout, sys.stderr, api_client=stubs.api)
1213         self.assertEqual(exited, 0)
1214
1215         body = {
1216             "workflow": {
1217                 "name": "submit_wf.cwl",
1218                 "description": "",
1219                 "definition": self.expect_workflow,
1220             }
1221         }
1222         stubs.api.workflows().update.assert_called_with(
1223             uuid=self.existing_workflow_uuid,
1224             body=JsonDiffMatcher(body))
1225         self.assertEqual(capture_stdout.getvalue(),
1226                          self.existing_workflow_uuid + '\n')
1227
1228
1229     @stubs
1230     def test_update_name(self, stubs):
1231         capture_stdout = cStringIO.StringIO()
1232
1233         exited = arvados_cwl.main(
1234             ["--update-workflow", self.existing_workflow_uuid,
1235              "--debug", "--name", "testing 123",
1236              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1237             capture_stdout, sys.stderr, api_client=stubs.api)
1238         self.assertEqual(exited, 0)
1239
1240         body = {
1241             "workflow": {
1242                 "name": "testing 123",
1243                 "description": "",
1244                 "definition": self.expect_workflow,
1245             }
1246         }
1247         stubs.api.workflows().update.assert_called_with(
1248             uuid=self.existing_workflow_uuid,
1249             body=JsonDiffMatcher(body))
1250         self.assertEqual(capture_stdout.getvalue(),
1251                          self.existing_workflow_uuid + '\n')
1252
1253
1254     @stubs
1255     def test_create_collection_per_tool(self, stubs):
1256         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1257
1258         capture_stdout = cStringIO.StringIO()
1259
1260         exited = arvados_cwl.main(
1261             ["--create-workflow", "--debug",
1262              "--api=containers",
1263              "--project-uuid", project_uuid,
1264              "tests/collection_per_tool/collection_per_tool.cwl"],
1265             capture_stdout, sys.stderr, api_client=stubs.api)
1266         self.assertEqual(exited, 0)
1267
1268         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1269         expect_workflow = StripYAMLComments(open(toolfile).read())
1270
1271         body = {
1272             "workflow": {
1273                 "owner_uuid": project_uuid,
1274                 "name": "collection_per_tool.cwl",
1275                 "description": "",
1276                 "definition": expect_workflow,
1277             }
1278         }
1279         stubs.api.workflows().create.assert_called_with(
1280             body=JsonDiffMatcher(body))
1281
1282         self.assertEqual(capture_stdout.getvalue(),
1283                          stubs.expect_workflow_uuid + '\n')
1284
1285 class TestTemplateInputs(unittest.TestCase):
1286     expect_template = {
1287         "components": {
1288             "inputs_test.cwl": {
1289                 'runtime_constraints': {
1290                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1291                     'min_ram_mb_per_node': 1024
1292                 },
1293                 'script_parameters': {
1294                     'cwl:tool':
1295                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1296                     'optionalFloatInput': None,
1297                     'fileInput': {
1298                         'type': 'File',
1299                         'dataclass': 'File',
1300                         'required': True,
1301                         'title': "It's a file; we expect to find some characters in it.",
1302                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1303                     },
1304                     'floatInput': {
1305                         'type': 'float',
1306                         'dataclass': 'number',
1307                         'required': True,
1308                         'title': 'Floats like a duck',
1309                         'default': 0.1,
1310                         'value': 0.1,
1311                     },
1312                     'optionalFloatInput': {
1313                         'type': ['null', 'float'],
1314                         'dataclass': 'number',
1315                         'required': False,
1316                     },
1317                     'boolInput': {
1318                         'type': 'boolean',
1319                         'dataclass': 'boolean',
1320                         'required': True,
1321                         'title': 'True or false?',
1322                     },
1323                 },
1324                 'repository': 'arvados',
1325                 'script_version': 'master',
1326                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1327                 'script': 'cwl-runner',
1328             },
1329         },
1330         "name": "inputs_test.cwl",
1331     }
1332
1333     @stubs
1334     def test_inputs_empty(self, stubs):
1335         exited = arvados_cwl.main(
1336             ["--create-template",
1337              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1338             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1339         self.assertEqual(exited, 0)
1340
1341         stubs.api.pipeline_templates().create.assert_called_with(
1342             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1343
1344     @stubs
1345     def test_inputs(self, stubs):
1346         exited = arvados_cwl.main(
1347             ["--create-template",
1348              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1349             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1350         self.assertEqual(exited, 0)
1351
1352         expect_template = copy.deepcopy(self.expect_template)
1353         params = expect_template[
1354             "components"]["inputs_test.cwl"]["script_parameters"]
1355         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1356         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1357         params["floatInput"]["value"] = 1.234
1358         params["boolInput"]["value"] = True
1359
1360         stubs.api.pipeline_templates().create.assert_called_with(
1361             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)