sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:debug': True,
 172                         'arv:enable_reuse': True,
 173                         'arv:on_error': 'continue'
 174                     },
 175                     'repository': 'arvados',
 176                     'script_version': 'master',
 177                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 178                     'script': 'cwl-runner',
 179                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 180                 }
 181             }
 182         }
 183         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 184         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 185         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 186         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 187         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 188             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 189             "state": "Queued"
 190         }
 191         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 192         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 193
 194         with open("tests/wf/submit_wf_packed.cwl") as f:
 195             expect_packed_workflow = yaml.round_trip_load(f)
 196
 197         stubs.expect_container_spec = {
 198             'priority': 500,
 199             'mounts': {
 200                 '/var/spool/cwl': {
 201                     'writable': True,
 202                     'kind': 'collection'
 203                 },
 204                 '/var/lib/cwl/workflow.json': {
 205                     'content': expect_packed_workflow,
 206                     'kind': 'json'
 207                 },
 208                 'stdout': {
 209                     'path': '/var/spool/cwl/cwl.output.json',
 210                     'kind': 'file'
 211                 },
 212                 '/var/lib/cwl/cwl.input.json': {
 213                     'kind': 'json',
 214                     'content': {
 215                         'y': {
 216                             'basename': '99999999999999999999999999999998+99',
 217                             'location': 'keep:99999999999999999999999999999998+99',
 218                             'class': 'Directory'},
 219                         'x': {
 220                             'basename': u'blorp.txt',
 221                             'class': 'File',
 222                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 223                             "size": 16
 224                         },
 225                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 226                             {'basename': 'renamed.txt',
 227                              'class': 'File',
 228                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 229                             }
 230                         ]}
 231                     },
 232                     'kind': 'json'
 233                 }
 234             },
 235             'secret_mounts': {},
 236             'state': 'Committed',
 237             'owner_uuid': None,
 238             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 239                         '--no-log-timestamps', '--disable-validate',
 240                         '--eval-timeout=20', '--thread-count=4',
 241                         '--enable-reuse', '--debug', '--on-error=continue',
 242                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 243             'name': 'submit_wf.cwl',
 244             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 245             'output_path': '/var/spool/cwl',
 246             'cwd': '/var/spool/cwl',
 247             'runtime_constraints': {
 248                 'API': True,
 249                 'vcpus': 1,
 250                 'ram': 1024*1024*1024
 251             },
 252             'use_existing': True,
 253             'properties': {},
 254             'secret_mounts': {}
 255         }
 256
 257         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 258         stubs.api.workflows().create().execute.return_value = {
 259             "uuid": stubs.expect_workflow_uuid,
 260         }
 261         def update_mock(**kwargs):
 262             stubs.updated_uuid = kwargs.get('uuid')
 263             return mock.DEFAULT
 264         stubs.api.workflows().update.side_effect = update_mock
 265         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 266             "uuid": stubs.updated_uuid,
 267         }
 268
 269         return func(self, stubs, *args, **kwargs)
 270     return wrapped
 271
 272
 273 class TestSubmit(unittest.TestCase):
 274     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 275     @mock.patch("time.sleep")
 276     @stubs
 277     def test_submit(self, stubs, tm, arvdock):
 278         capture_stdout = cStringIO.StringIO()
 279         exited = arvados_cwl.main(
 280             ["--submit", "--no-wait", "--api=jobs", "--debug",
 281              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 282             capture_stdout, sys.stderr, api_client=stubs.api)
 283         self.assertEqual(exited, 0)
 284
 285         stubs.api.collections().create.assert_has_calls([
 286             mock.call(body=JsonDiffMatcher({
 287                 'manifest_text':
 288                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 289                 'replication_desired': None,
 290                 'name': 'submit_tool.cwl dependencies',
 291             }), ensure_unique_name=True),
 292             mock.call(body=JsonDiffMatcher({
 293                 'manifest_text':
 294                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 295                 'replication_desired': None,
 296                 'name': 'submit_wf.cwl input',
 297             }), ensure_unique_name=True),
 298             mock.call(body=JsonDiffMatcher({
 299                 'manifest_text':
 300                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 301                 'replication_desired': None,
 302                 'name': 'submit_wf.cwl',
 303             }), ensure_unique_name=True)        ])
 304
 305         arvdock.assert_has_calls([
 306             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 307             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 308         ])
 309
 310         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 311         stubs.api.pipeline_instances().create.assert_called_with(
 312             body=JsonDiffMatcher(expect_pipeline))
 313         self.assertEqual(capture_stdout.getvalue(),
 314                          stubs.expect_pipeline_uuid + '\n')
 315
 316
 317     @mock.patch("time.sleep")
 318     @stubs
 319     def test_submit_no_reuse(self, stubs, tm):
 320         capture_stdout = cStringIO.StringIO()
 321         exited = arvados_cwl.main(
 322             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 323              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 324             capture_stdout, sys.stderr, api_client=stubs.api)
 325         self.assertEqual(exited, 0)
 326
 327         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 328         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 329         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 330
 331         stubs.api.pipeline_instances().create.assert_called_with(
 332             body=JsonDiffMatcher(expect_pipeline))
 333         self.assertEqual(capture_stdout.getvalue(),
 334                          stubs.expect_pipeline_uuid + '\n')
 335
 336     @mock.patch("time.sleep")
 337     @stubs
 338     def test_submit_on_error(self, stubs, tm):
 339         capture_stdout = cStringIO.StringIO()
 340         exited = arvados_cwl.main(
 341             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 342              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 343             capture_stdout, sys.stderr, api_client=stubs.api)
 344         self.assertEqual(exited, 0)
 345
 346         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 347         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 348
 349         stubs.api.pipeline_instances().create.assert_called_with(
 350             body=JsonDiffMatcher(expect_pipeline))
 351         self.assertEqual(capture_stdout.getvalue(),
 352                          stubs.expect_pipeline_uuid + '\n')
 353
 354
 355     @mock.patch("time.sleep")
 356     @stubs
 357     def test_submit_runner_ram(self, stubs, tm):
 358         capture_stdout = cStringIO.StringIO()
 359         exited = arvados_cwl.main(
 360             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 361              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 362             capture_stdout, sys.stderr, api_client=stubs.api)
 363         self.assertEqual(exited, 0)
 364
 365         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 366         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 367
 368         stubs.api.pipeline_instances().create.assert_called_with(
 369             body=JsonDiffMatcher(expect_pipeline))
 370         self.assertEqual(capture_stdout.getvalue(),
 371                          stubs.expect_pipeline_uuid + '\n')
 372
 373
 374     @mock.patch("time.sleep")
 375     @stubs
 376     def test_submit_invalid_runner_ram(self, stubs, tm):
 377         capture_stdout = cStringIO.StringIO()
 378         exited = arvados_cwl.main(
 379             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 380              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 381             capture_stdout, sys.stderr, api_client=stubs.api)
 382         self.assertEqual(exited, 1)
 383
 384     @mock.patch("time.sleep")
 385     @stubs
 386     def test_submit_output_name(self, stubs, tm):
 387         output_name = "test_output_name"
 388
 389         capture_stdout = cStringIO.StringIO()
 390         exited = arvados_cwl.main(
 391             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 392              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 393             capture_stdout, sys.stderr, api_client=stubs.api)
 394         self.assertEqual(exited, 0)
 395
 396         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 397         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 398
 399         stubs.api.pipeline_instances().create.assert_called_with(
 400             body=JsonDiffMatcher(expect_pipeline))
 401         self.assertEqual(capture_stdout.getvalue(),
 402                          stubs.expect_pipeline_uuid + '\n')
 403
 404
 405     @mock.patch("time.sleep")
 406     @stubs
 407     def test_submit_pipeline_name(self, stubs, tm):
 408         capture_stdout = cStringIO.StringIO()
 409         exited = arvados_cwl.main(
 410             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 411              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 412             capture_stdout, sys.stderr, api_client=stubs.api)
 413         self.assertEqual(exited, 0)
 414
 415         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 416         expect_pipeline["name"] = "hello job 123"
 417
 418         stubs.api.pipeline_instances().create.assert_called_with(
 419             body=JsonDiffMatcher(expect_pipeline))
 420         self.assertEqual(capture_stdout.getvalue(),
 421                          stubs.expect_pipeline_uuid + '\n')
 422
 423     @mock.patch("time.sleep")
 424     @stubs
 425     def test_submit_output_tags(self, stubs, tm):
 426         output_tags = "tag0,tag1,tag2"
 427
 428         capture_stdout = cStringIO.StringIO()
 429         exited = arvados_cwl.main(
 430             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 431              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 432             capture_stdout, sys.stderr, api_client=stubs.api)
 433         self.assertEqual(exited, 0)
 434
 435         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 436         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 437
 438         stubs.api.pipeline_instances().create.assert_called_with(
 439             body=JsonDiffMatcher(expect_pipeline))
 440         self.assertEqual(capture_stdout.getvalue(),
 441                          stubs.expect_pipeline_uuid + '\n')
 442
 443     @mock.patch("time.sleep")
 444     @stubs
 445     def test_submit_with_project_uuid(self, stubs, tm):
 446         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 447
 448         exited = arvados_cwl.main(
 449             ["--submit", "--no-wait", "--debug",
 450              "--project-uuid", project_uuid,
 451              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 452             sys.stdout, sys.stderr, api_client=stubs.api)
 453         self.assertEqual(exited, 0)
 454
 455         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 456         expect_pipeline["owner_uuid"] = project_uuid
 457         stubs.api.pipeline_instances().create.assert_called_with(
 458             body=JsonDiffMatcher(expect_pipeline))
 459
 460     @stubs
 461     def test_submit_container(self, stubs):
 462         capture_stdout = cStringIO.StringIO()
 463         try:
 464             exited = arvados_cwl.main(
 465                 ["--submit", "--no-wait", "--api=containers", "--debug",
 466                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468             self.assertEqual(exited, 0)
 469         except:
 470             logging.exception("")
 471
 472         stubs.api.collections().create.assert_has_calls([
 473             mock.call(body=JsonDiffMatcher({
 474                 'manifest_text':
 475                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 476                 'replication_desired': None,
 477                 'name': 'submit_tool.cwl dependencies',
 478             }), ensure_unique_name=True),
 479             mock.call(body=JsonDiffMatcher({
 480                 'manifest_text':
 481                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 482                 'replication_desired': None,
 483                 'name': 'submit_wf.cwl input',
 484             }), ensure_unique_name=True)])
 485
 486         expect_container = copy.deepcopy(stubs.expect_container_spec)
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_container))
 489         self.assertEqual(capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491
 492     @stubs
 493     def test_submit_container_no_reuse(self, stubs):
 494         capture_stdout = cStringIO.StringIO()
 495         try:
 496             exited = arvados_cwl.main(
 497                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 498                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 499                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 500             self.assertEqual(exited, 0)
 501         except:
 502             logging.exception("")
 503
 504         expect_container = copy.deepcopy(stubs.expect_container_spec)
 505         expect_container["command"] = [
 506             'arvados-cwl-runner', '--local', '--api=containers',
 507             '--no-log-timestamps', '--disable-validate',
 508             '--eval-timeout=20', '--thread-count=4',
 509             '--disable-reuse', '--debug', '--on-error=continue',
 510             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 511         expect_container["use_existing"] = False
 512
 513         stubs.api.container_requests().create.assert_called_with(
 514             body=JsonDiffMatcher(expect_container))
 515         self.assertEqual(capture_stdout.getvalue(),
 516                          stubs.expect_container_request_uuid + '\n')
 517
 518
 519     @stubs
 520     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 521         capture_stdout = cStringIO.StringIO()
 522
 523         exited = arvados_cwl.main(
 524             ["--submit", "--no-wait", "--api=containers", "--debug",
 525              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 526             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 527         self.assertEqual(exited, 0)
 528
 529         expect_container = copy.deepcopy(stubs.expect_container_spec)
 530         expect_container["command"] = [
 531             'arvados-cwl-runner', '--local', '--api=containers',
 532             '--no-log-timestamps', '--disable-validate',
 533             '--eval-timeout=20', '--thread-count=4',
 534             '--disable-reuse', '--debug', '--on-error=continue',
 535             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 536         expect_container["use_existing"] = False
 537         expect_container["name"] = "submit_wf_no_reuse.cwl"
 538         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 539             {
 540                 "class": "http://arvados.org/cwl#ReuseRequirement",
 541                 "enableReuse": False,
 542             },
 543         ]
 544         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 545             "arv": "http://arvados.org/cwl#",
 546             "cwltool": "http://commonwl.org/cwltool#"
 547         }
 548
 549         stubs.api.container_requests().create.assert_called_with(
 550             body=JsonDiffMatcher(expect_container))
 551         self.assertEqual(capture_stdout.getvalue(),
 552                          stubs.expect_container_request_uuid + '\n')
 553
 554
 555     @stubs
 556     def test_submit_container_on_error(self, stubs):
 557         capture_stdout = cStringIO.StringIO()
 558         try:
 559             exited = arvados_cwl.main(
 560                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 561                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 562                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 563             self.assertEqual(exited, 0)
 564         except:
 565             logging.exception("")
 566
 567         expect_container = copy.deepcopy(stubs.expect_container_spec)
 568         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 569                                        '--no-log-timestamps', '--disable-validate',
 570                                        '--eval-timeout=20', '--thread-count=4',
 571                                        '--enable-reuse', '--debug', '--on-error=stop',
 572                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 573
 574         stubs.api.container_requests().create.assert_called_with(
 575             body=JsonDiffMatcher(expect_container))
 576         self.assertEqual(capture_stdout.getvalue(),
 577                          stubs.expect_container_request_uuid + '\n')
 578
 579     @stubs
 580     def test_submit_container_output_name(self, stubs):
 581         output_name = "test_output_name"
 582
 583         capture_stdout = cStringIO.StringIO()
 584         try:
 585             exited = arvados_cwl.main(
 586                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 587                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 588                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 589             self.assertEqual(exited, 0)
 590         except:
 591             logging.exception("")
 592
 593         expect_container = copy.deepcopy(stubs.expect_container_spec)
 594         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 595                                        '--no-log-timestamps', '--disable-validate',
 596                                        '--eval-timeout=20', '--thread-count=4',
 597                                        '--enable-reuse',
 598                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 599                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 600         expect_container["output_name"] = output_name
 601
 602         stubs.api.container_requests().create.assert_called_with(
 603             body=JsonDiffMatcher(expect_container))
 604         self.assertEqual(capture_stdout.getvalue(),
 605                          stubs.expect_container_request_uuid + '\n')
 606
 607
 608     @stubs
 609     def test_submit_container_output_ttl(self, stubs):
 610         capture_stdout = cStringIO.StringIO()
 611         try:
 612             exited = arvados_cwl.main(
 613                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 614                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 615                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 616             self.assertEqual(exited, 0)
 617         except:
 618             logging.exception("")
 619
 620         expect_container = copy.deepcopy(stubs.expect_container_spec)
 621         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 622                                        '--no-log-timestamps', '--disable-validate',
 623                                        '--eval-timeout=20', '--thread-count=4',
 624                                        '--enable-reuse', '--debug', '--on-error=continue',
 625                                        "--intermediate-output-ttl=3600",
 626                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 627
 628         stubs.api.container_requests().create.assert_called_with(
 629             body=JsonDiffMatcher(expect_container))
 630         self.assertEqual(capture_stdout.getvalue(),
 631                          stubs.expect_container_request_uuid + '\n')
 632
 633     @stubs
 634     def test_submit_container_trash_intermediate(self, stubs):
 635         capture_stdout = cStringIO.StringIO()
 636         try:
 637             exited = arvados_cwl.main(
 638                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 639                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 640                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 641             self.assertEqual(exited, 0)
 642         except:
 643             logging.exception("")
 644
 645         expect_container = copy.deepcopy(stubs.expect_container_spec)
 646         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 647                                        '--no-log-timestamps', '--disable-validate',
 648                                        '--eval-timeout=20', '--thread-count=4',
 649                                        '--enable-reuse', '--debug', '--on-error=continue',
 650                                        "--trash-intermediate",
 651                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 652
 653         stubs.api.container_requests().create.assert_called_with(
 654             body=JsonDiffMatcher(expect_container))
 655         self.assertEqual(capture_stdout.getvalue(),
 656                          stubs.expect_container_request_uuid + '\n')
 657
 658     @stubs
 659     def test_submit_container_output_tags(self, stubs):
 660         output_tags = "tag0,tag1,tag2"
 661
 662         capture_stdout = cStringIO.StringIO()
 663         try:
 664             exited = arvados_cwl.main(
 665                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 666                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 667                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 668             self.assertEqual(exited, 0)
 669         except:
 670             logging.exception("")
 671
 672         expect_container = copy.deepcopy(stubs.expect_container_spec)
 673         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 674                                        '--no-log-timestamps', '--disable-validate',
 675                                        '--eval-timeout=20', '--thread-count=4',
 676                                        '--enable-reuse',
 677                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 678                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 679
 680         stubs.api.container_requests().create.assert_called_with(
 681             body=JsonDiffMatcher(expect_container))
 682         self.assertEqual(capture_stdout.getvalue(),
 683                          stubs.expect_container_request_uuid + '\n')
 684
 685     @stubs
 686     def test_submit_container_runner_ram(self, stubs):
 687         capture_stdout = cStringIO.StringIO()
 688         try:
 689             exited = arvados_cwl.main(
 690                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 691                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 692                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 693             self.assertEqual(exited, 0)
 694         except:
 695             logging.exception("")
 696
 697         expect_container = copy.deepcopy(stubs.expect_container_spec)
 698         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 699
 700         stubs.api.container_requests().create.assert_called_with(
 701             body=JsonDiffMatcher(expect_container))
 702         self.assertEqual(capture_stdout.getvalue(),
 703                          stubs.expect_container_request_uuid + '\n')
 704
 705     @mock.patch("arvados.collection.CollectionReader")
 706     @mock.patch("time.sleep")
 707     @stubs
 708     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 709         capture_stdout = cStringIO.StringIO()
 710         exited = arvados_cwl.main(
 711             ["--submit", "--no-wait", "--api=containers", "--debug",
 712              "tests/wf/submit_keepref_wf.cwl"],
 713             capture_stdout, sys.stderr, api_client=stubs.api)
 714         self.assertEqual(exited, 0)
 715
 716
 717     @mock.patch("arvados.collection.CollectionReader")
 718     @mock.patch("time.sleep")
 719     @stubs
 720     def test_submit_keepref(self, stubs, tm, reader):
 721         capture_stdout = cStringIO.StringIO()
 722
 723         with open("tests/wf/expect_arvworkflow.cwl") as f:
 724             reader().open().__enter__().read.return_value = f.read()
 725
 726         exited = arvados_cwl.main(
 727             ["--submit", "--no-wait", "--api=containers", "--debug",
 728              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 729             capture_stdout, sys.stderr, api_client=stubs.api)
 730         self.assertEqual(exited, 0)
 731
 732         expect_container = {
 733             'priority': 500,
 734             'mounts': {
 735                 '/var/spool/cwl': {
 736                     'writable': True,
 737                     'kind': 'collection'
 738                 },
 739                 'stdout': {
 740                     'path': '/var/spool/cwl/cwl.output.json',
 741                     'kind': 'file'
 742                 },
 743                 '/var/lib/cwl/workflow': {
 744                     'portable_data_hash': '99999999999999999999999999999994+99',
 745                     'kind': 'collection'
 746                 },
 747                 '/var/lib/cwl/cwl.input.json': {
 748                     'content': {
 749                         'x': 'XxX'
 750                     },
 751                     'kind': 'json'
 752                 }
 753             }, 'state': 'Committed',
 754             'owner_uuid': None,
 755             'output_path': '/var/spool/cwl',
 756             'name': 'expect_arvworkflow.cwl#main',
 757             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 758             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 759                         '--no-log-timestamps', '--disable-validate',
 760                         '--eval-timeout=20', '--thread-count=4',
 761                         '--enable-reuse', '--debug', '--on-error=continue',
 762                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 763             'cwd': '/var/spool/cwl',
 764             'runtime_constraints': {
 765                 'API': True,
 766                 'vcpus': 1,
 767                 'ram': 1073741824
 768             },
 769             'use_existing': True,
 770             'properties': {},
 771             'secret_mounts': {}
 772         }
 773
 774         stubs.api.container_requests().create.assert_called_with(
 775             body=JsonDiffMatcher(expect_container))
 776         self.assertEqual(capture_stdout.getvalue(),
 777                          stubs.expect_container_request_uuid + '\n')
 778
 779
 780     @mock.patch("arvados.collection.CollectionReader")
 781     @mock.patch("time.sleep")
 782     @stubs
 783     def test_submit_jobs_keepref(self, stubs, tm, reader):
 784         capture_stdout = cStringIO.StringIO()
 785
 786         with open("tests/wf/expect_arvworkflow.cwl") as f:
 787             reader().open().__enter__().read.return_value = f.read()
 788
 789         exited = arvados_cwl.main(
 790             ["--submit", "--no-wait", "--api=jobs", "--debug",
 791              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 792             capture_stdout, sys.stderr, api_client=stubs.api)
 793         self.assertEqual(exited, 0)
 794
 795         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 796         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 797         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 798         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 799         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 800         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 801         stubs.api.pipeline_instances().create.assert_called_with(
 802             body=JsonDiffMatcher(expect_pipeline))
 803
 804     @mock.patch("time.sleep")
 805     @stubs
 806     def test_submit_arvworkflow(self, stubs, tm):
 807         capture_stdout = cStringIO.StringIO()
 808
 809         with open("tests/wf/expect_arvworkflow.cwl") as f:
 810             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 811
 812         exited = arvados_cwl.main(
 813             ["--submit", "--no-wait", "--api=containers", "--debug",
 814              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 815             capture_stdout, sys.stderr, api_client=stubs.api)
 816         self.assertEqual(exited, 0)
 817
 818         expect_container = {
 819             'priority': 500,
 820             'mounts': {
 821                 '/var/spool/cwl': {
 822                     'writable': True,
 823                     'kind': 'collection'
 824                 },
 825                 'stdout': {
 826                     'path': '/var/spool/cwl/cwl.output.json',
 827                     'kind': 'file'
 828                 },
 829                 '/var/lib/cwl/workflow.json': {
 830                     'kind': 'json',
 831                     'content': {
 832                         'cwlVersion': 'v1.0',
 833                         '$graph': [
 834                             {
 835                                 'id': '#main',
 836                                 'inputs': [
 837                                     {'type': 'string', 'id': '#main/x'}
 838                                 ],
 839                                 'steps': [
 840                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 841                                      'run': '#submit_tool.cwl',
 842                                      'id': '#main/step1',
 843                                      'out': []}
 844                                 ],
 845                                 'class': 'Workflow',
 846                                 'outputs': []
 847                             },
 848                             {
 849                                 'inputs': [
 850                                     {
 851                                         'inputBinding': {'position': 1},
 852                                         'type': 'string',
 853                                         'id': '#submit_tool.cwl/x'}
 854                                 ],
 855                                 'requirements': [
 856                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 857                                 ],
 858                                 'id': '#submit_tool.cwl',
 859                                 'outputs': [],
 860                                 'baseCommand': 'cat',
 861                                 'class': 'CommandLineTool'
 862                             }
 863                         ]
 864                     }
 865                 },
 866                 '/var/lib/cwl/cwl.input.json': {
 867                     'content': {
 868                         'x': 'XxX'
 869                     },
 870                     'kind': 'json'
 871                 }
 872             }, 'state': 'Committed',
 873             'owner_uuid': None,
 874             'output_path': '/var/spool/cwl',
 875             'name': 'a test workflow',
 876             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 877             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 878                         '--no-log-timestamps', '--disable-validate',
 879                         '--eval-timeout=20', '--thread-count=4',
 880                         '--enable-reuse', '--debug', '--on-error=continue',
 881                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 882             'cwd': '/var/spool/cwl',
 883             'runtime_constraints': {
 884                 'API': True,
 885                 'vcpus': 1,
 886                 'ram': 1073741824
 887             },
 888             'use_existing': True,
 889             'properties': {
 890                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 891             },
 892             'secret_mounts': {}
 893         }
 894
 895         stubs.api.container_requests().create.assert_called_with(
 896             body=JsonDiffMatcher(expect_container))
 897         self.assertEqual(capture_stdout.getvalue(),
 898                          stubs.expect_container_request_uuid + '\n')
 899
 900
 901     @stubs
 902     def test_submit_container_name(self, stubs):
 903         capture_stdout = cStringIO.StringIO()
 904         try:
 905             exited = arvados_cwl.main(
 906                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 907                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 908                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 909             self.assertEqual(exited, 0)
 910         except:
 911             logging.exception("")
 912
 913         expect_container = copy.deepcopy(stubs.expect_container_spec)
 914         expect_container["name"] = "hello container 123"
 915
 916         stubs.api.container_requests().create.assert_called_with(
 917             body=JsonDiffMatcher(expect_container))
 918         self.assertEqual(capture_stdout.getvalue(),
 919                          stubs.expect_container_request_uuid + '\n')
 920
 921
 922     @stubs
 923     def test_submit_container_project(self, stubs):
 924         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 925         capture_stdout = cStringIO.StringIO()
 926         try:
 927             exited = arvados_cwl.main(
 928                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 929                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 930                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 931             self.assertEqual(exited, 0)
 932         except:
 933             logging.exception("")
 934
 935         expect_container = copy.deepcopy(stubs.expect_container_spec)
 936         expect_container["owner_uuid"] = project_uuid
 937         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 938                                        '--no-log-timestamps', '--disable-validate',
 939                                        "--eval-timeout=20", "--thread-count=4",
 940                                        '--enable-reuse', '--debug', '--on-error=continue',
 941                                        '--project-uuid='+project_uuid,
 942                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 943
 944         stubs.api.container_requests().create.assert_called_with(
 945             body=JsonDiffMatcher(expect_container))
 946         self.assertEqual(capture_stdout.getvalue(),
 947                          stubs.expect_container_request_uuid + '\n')
 948
 949     @stubs
 950     def test_submit_container_eval_timeout(self, stubs):
 951         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 952         capture_stdout = cStringIO.StringIO()
 953         try:
 954             exited = arvados_cwl.main(
 955                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 956                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 957                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 958             self.assertEqual(exited, 0)
 959         except:
 960             logging.exception("")
 961
 962         expect_container = copy.deepcopy(stubs.expect_container_spec)
 963         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 964                                        '--no-log-timestamps', '--disable-validate',
 965                                        '--eval-timeout=60.0', '--thread-count=4',
 966                                        '--enable-reuse', '--debug', '--on-error=continue',
 967                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 968
 969         stubs.api.container_requests().create.assert_called_with(
 970             body=JsonDiffMatcher(expect_container))
 971         self.assertEqual(capture_stdout.getvalue(),
 972                          stubs.expect_container_request_uuid + '\n')
 973
 974
 975     @stubs
 976     def test_submit_container_thread_count(self, stubs):
 977         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 978         capture_stdout = cStringIO.StringIO()
 979         try:
 980             exited = arvados_cwl.main(
 981                 ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 982                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 983                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 984             self.assertEqual(exited, 0)
 985         except:
 986             logging.exception("")
 987
 988         expect_container = copy.deepcopy(stubs.expect_container_spec)
 989         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 990                                        '--no-log-timestamps', '--disable-validate',
 991                                        '--eval-timeout=20', '--thread-count=20',
 992                                        '--enable-reuse', '--debug', '--on-error=continue',
 993                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 994
 995         stubs.api.container_requests().create.assert_called_with(
 996             body=JsonDiffMatcher(expect_container))
 997         self.assertEqual(capture_stdout.getvalue(),
 998                          stubs.expect_container_request_uuid + '\n')
 999
1000
1001     @stubs
1002     def test_submit_job_runner_image(self, stubs):
1003         capture_stdout = cStringIO.StringIO()
1004         try:
1005             exited = arvados_cwl.main(
1006                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1007                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1008                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1009             self.assertEqual(exited, 0)
1010         except:
1011             logging.exception("")
1012
1013         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
1014
1015         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1016         stubs.api.pipeline_instances().create.assert_called_with(
1017             body=JsonDiffMatcher(expect_pipeline))
1018         self.assertEqual(capture_stdout.getvalue(),
1019                          stubs.expect_pipeline_uuid + '\n')
1020
1021     @stubs
1022     def test_submit_container_runner_image(self, stubs):
1023         capture_stdout = cStringIO.StringIO()
1024         try:
1025             exited = arvados_cwl.main(
1026                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1027                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1028                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1029             self.assertEqual(exited, 0)
1030         except:
1031             logging.exception("")
1032
1033         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
1034
1035         expect_container = copy.deepcopy(stubs.expect_container_spec)
1036         stubs.api.container_requests().create.assert_called_with(
1037             body=JsonDiffMatcher(expect_container))
1038         self.assertEqual(capture_stdout.getvalue(),
1039                          stubs.expect_container_request_uuid + '\n')
1040
1041     @stubs
1042     def test_submit_priority(self, stubs):
1043         capture_stdout = cStringIO.StringIO()
1044         try:
1045             exited = arvados_cwl.main(
1046                 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1047                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1048                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1049             self.assertEqual(exited, 0)
1050         except:
1051             logging.exception("")
1052
1053         stubs.expect_container_spec["priority"] = 669
1054
1055         expect_container = copy.deepcopy(stubs.expect_container_spec)
1056         stubs.api.container_requests().create.assert_called_with(
1057             body=JsonDiffMatcher(expect_container))
1058         self.assertEqual(capture_stdout.getvalue(),
1059                          stubs.expect_container_request_uuid + '\n')
1060
1061
1062     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1063     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1064     @mock.patch("arvados.api")
1065     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1066         arvrunner = mock.MagicMock()
1067         arvrunner.project_uuid = ""
1068         api.return_value = mock.MagicMock()
1069         arvrunner.api = api.return_value
1070         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1071                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1072                                                                         "link_class": "docker_image_repo+tag",
1073                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1074                                                                         "owner_uuid": "",
1075                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1076                                                             {"items": [{"created_at": "",
1077                                                                         "head_uuid": "",
1078                                                                         "link_class": "docker_image_hash",
1079                                                                         "name": "123456",
1080                                                                         "owner_uuid": "",
1081                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1082         )
1083         find_one_image_hash.return_value = "123456"
1084
1085         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1086                                                                               "owner_uuid": "",
1087                                                                               "manifest_text": "",
1088                                                                               "properties": ""
1089                                                                           }], "items_available": 1, "offset": 0},)
1090         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1091         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1092                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1093
1094     @stubs
1095     def test_submit_secrets(self, stubs):
1096         capture_stdout = cStringIO.StringIO()
1097         try:
1098             exited = arvados_cwl.main(
1099                 ["--submit", "--no-wait", "--api=containers", "--debug",
1100                  "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1101                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1102             self.assertEqual(exited, 0)
1103         except:
1104             logging.exception("")
1105
1106
1107         expect_container = {
1108             "command": [
1109                 "arvados-cwl-runner",
1110                 "--local",
1111                 "--api=containers",
1112                 "--no-log-timestamps",
1113                 "--disable-validate",
1114                 "--eval-timeout=20",
1115                 '--thread-count=4',
1116                 "--enable-reuse",
1117                 '--debug',
1118                 "--on-error=continue",
1119                 "/var/lib/cwl/workflow.json#main",
1120                 "/var/lib/cwl/cwl.input.json"
1121             ],
1122             "container_image": "arvados/jobs:"+arvados_cwl.__version__,
1123             "cwd": "/var/spool/cwl",
1124             "mounts": {
1125                 "/var/lib/cwl/cwl.input.json": {
1126                     "content": {
1127                         "pw": {
1128                             "$include": "/secrets/s0"
1129                         }
1130                     },
1131                     "kind": "json"
1132                 },
1133                 "/var/lib/cwl/workflow.json": {
1134                     "content": {
1135                         "$graph": [
1136                             {
1137                                 "$namespaces": {
1138                                     "cwltool": "http://commonwl.org/cwltool#"
1139                                 },
1140                                 "arguments": [
1141                                     "md5sum",
1142                                     "example.conf"
1143                                 ],
1144                                 "class": "CommandLineTool",
1145                                 "hints": [
1146                                     {
1147                                         "class": "http://commonwl.org/cwltool#Secrets",
1148                                         "secrets": [
1149                                             "#secret_job.cwl/pw"
1150                                         ]
1151                                     }
1152                                 ],
1153                                 "id": "#secret_job.cwl",
1154                                 "inputs": [
1155                                     {
1156                                         "id": "#secret_job.cwl/pw",
1157                                         "type": "string"
1158                                     }
1159                                 ],
1160                                 "outputs": [
1161                                     {
1162                                         "id": "#secret_job.cwl/out",
1163                                         "type": "stdout"
1164                                     }
1165                                 ],
1166                                 "stdout": "hashed_example.txt",
1167                                 "requirements": [
1168                                     {
1169                                         "class": "InitialWorkDirRequirement",
1170                                         "listing": [
1171                                             {
1172                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1173                                                 "entryname": "example.conf"
1174                                             }
1175                                         ]
1176                                     }
1177                                 ]
1178                             },
1179                             {
1180                                 "class": "Workflow",
1181                                 "hints": [
1182                                     {
1183                                         "class": "DockerRequirement",
1184                                         "dockerPull": "debian:8"
1185                                     },
1186                                     {
1187                                         "class": "http://commonwl.org/cwltool#Secrets",
1188                                         "secrets": [
1189                                             "#main/pw"
1190                                         ]
1191                                     }
1192                                 ],
1193                                 "id": "#main",
1194                                 "inputs": [
1195                                     {
1196                                         "id": "#main/pw",
1197                                         "type": "string"
1198                                     }
1199                                 ],
1200                                 "outputs": [
1201                                     {
1202                                         "id": "#main/out",
1203                                         "outputSource": "#main/step1/out",
1204                                         "type": "File"
1205                                     }
1206                                 ],
1207                                 "steps": [
1208                                     {
1209                                         "id": "#main/step1",
1210                                         "in": [
1211                                             {
1212                                                 "id": "#main/step1/pw",
1213                                                 "source": "#main/pw"
1214                                             }
1215                                         ],
1216                                         "out": [
1217                                             "#main/step1/out"
1218                                         ],
1219                                         "run": "#secret_job.cwl"
1220                                     }
1221                                 ]
1222                             }
1223                         ],
1224                         "cwlVersion": "v1.0"
1225                     },
1226                     "kind": "json"
1227                 },
1228                 "/var/spool/cwl": {
1229                     "kind": "collection",
1230                     "writable": True
1231                 },
1232                 "stdout": {
1233                     "kind": "file",
1234                     "path": "/var/spool/cwl/cwl.output.json"
1235                 }
1236             },
1237             "name": "secret_wf.cwl",
1238             "output_path": "/var/spool/cwl",
1239             "owner_uuid": None,
1240             "priority": 500,
1241             "properties": {},
1242             "runtime_constraints": {
1243                 "API": True,
1244                 "ram": 1073741824,
1245                 "vcpus": 1
1246             },
1247             "secret_mounts": {
1248                 "/secrets/s0": {
1249                     "content": "blorp",
1250                     "kind": "text"
1251                 }
1252             },
1253             "state": "Committed",
1254             "use_existing": True
1255         }
1256
1257         stubs.api.container_requests().create.assert_called_with(
1258             body=JsonDiffMatcher(expect_container))
1259         self.assertEqual(capture_stdout.getvalue(),
1260                          stubs.expect_container_request_uuid + '\n')
1261
1262
1263 class TestCreateTemplate(unittest.TestCase):
1264     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1265
1266     def _adjust_script_params(self, expect_component):
1267         expect_component['script_parameters']['x'] = {
1268             'dataclass': 'File',
1269             'required': True,
1270             'type': 'File',
1271             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1272         }
1273         expect_component['script_parameters']['y'] = {
1274             'dataclass': 'Collection',
1275             'required': True,
1276             'type': 'Directory',
1277             'value': '99999999999999999999999999999998+99',
1278         }
1279         expect_component['script_parameters']['z'] = {
1280             'dataclass': 'Collection',
1281             'required': True,
1282             'type': 'Directory',
1283         }
1284
1285     @stubs
1286     def test_create(self, stubs):
1287         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1288
1289         capture_stdout = cStringIO.StringIO()
1290
1291         exited = arvados_cwl.main(
1292             ["--create-workflow", "--debug",
1293              "--api=jobs",
1294              "--project-uuid", project_uuid,
1295              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1296             capture_stdout, sys.stderr, api_client=stubs.api)
1297         self.assertEqual(exited, 0)
1298
1299         stubs.api.pipeline_instances().create.refute_called()
1300         stubs.api.jobs().create.refute_called()
1301
1302         expect_component = copy.deepcopy(stubs.expect_job_spec)
1303         self._adjust_script_params(expect_component)
1304         expect_template = {
1305             "components": {
1306                 "submit_wf.cwl": expect_component,
1307             },
1308             "name": "submit_wf.cwl",
1309             "owner_uuid": project_uuid,
1310         }
1311         stubs.api.pipeline_templates().create.assert_called_with(
1312             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1313
1314         self.assertEqual(capture_stdout.getvalue(),
1315                          stubs.expect_pipeline_template_uuid + '\n')
1316
1317
1318     @stubs
1319     def test_create_name(self, stubs):
1320         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1321
1322         capture_stdout = cStringIO.StringIO()
1323
1324         exited = arvados_cwl.main(
1325             ["--create-workflow", "--debug",
1326              "--project-uuid", project_uuid,
1327              "--api=jobs",
1328              "--name", "testing 123",
1329              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1330             capture_stdout, sys.stderr, api_client=stubs.api)
1331         self.assertEqual(exited, 0)
1332
1333         stubs.api.pipeline_instances().create.refute_called()
1334         stubs.api.jobs().create.refute_called()
1335
1336         expect_component = copy.deepcopy(stubs.expect_job_spec)
1337         self._adjust_script_params(expect_component)
1338         expect_template = {
1339             "components": {
1340                 "testing 123": expect_component,
1341             },
1342             "name": "testing 123",
1343             "owner_uuid": project_uuid,
1344         }
1345         stubs.api.pipeline_templates().create.assert_called_with(
1346             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1347
1348         self.assertEqual(capture_stdout.getvalue(),
1349                          stubs.expect_pipeline_template_uuid + '\n')
1350
1351
1352     @stubs
1353     def test_update_name(self, stubs):
1354         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1355
1356         capture_stdout = cStringIO.StringIO()
1357
1358         exited = arvados_cwl.main(
1359             ["--update-workflow", self.existing_template_uuid,
1360              "--debug",
1361              "--project-uuid", project_uuid,
1362              "--api=jobs",
1363              "--name", "testing 123",
1364              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1365             capture_stdout, sys.stderr, api_client=stubs.api)
1366         self.assertEqual(exited, 0)
1367
1368         stubs.api.pipeline_instances().create.refute_called()
1369         stubs.api.jobs().create.refute_called()
1370
1371         expect_component = copy.deepcopy(stubs.expect_job_spec)
1372         self._adjust_script_params(expect_component)
1373         expect_template = {
1374             "components": {
1375                 "testing 123": expect_component,
1376             },
1377             "name": "testing 123",
1378             "owner_uuid": project_uuid,
1379         }
1380         stubs.api.pipeline_templates().create.refute_called()
1381         stubs.api.pipeline_templates().update.assert_called_with(
1382             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1383
1384         self.assertEqual(capture_stdout.getvalue(),
1385                          self.existing_template_uuid + '\n')
1386
1387
1388 class TestCreateWorkflow(unittest.TestCase):
1389     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1390     expect_workflow = StripYAMLComments(
1391         open("tests/wf/expect_packed.cwl").read())
1392
1393     @stubs
1394     def test_create(self, stubs):
1395         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1396
1397         capture_stdout = cStringIO.StringIO()
1398
1399         exited = arvados_cwl.main(
1400             ["--create-workflow", "--debug",
1401              "--api=containers",
1402              "--project-uuid", project_uuid,
1403              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1404             capture_stdout, sys.stderr, api_client=stubs.api)
1405         self.assertEqual(exited, 0)
1406
1407         stubs.api.pipeline_templates().create.refute_called()
1408         stubs.api.container_requests().create.refute_called()
1409
1410         body = {
1411             "workflow": {
1412                 "owner_uuid": project_uuid,
1413                 "name": "submit_wf.cwl",
1414                 "description": "",
1415                 "definition": self.expect_workflow,
1416             }
1417         }
1418         stubs.api.workflows().create.assert_called_with(
1419             body=JsonDiffMatcher(body))
1420
1421         self.assertEqual(capture_stdout.getvalue(),
1422                          stubs.expect_workflow_uuid + '\n')
1423
1424
1425     @stubs
1426     def test_create_name(self, stubs):
1427         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1428
1429         capture_stdout = cStringIO.StringIO()
1430
1431         exited = arvados_cwl.main(
1432             ["--create-workflow", "--debug",
1433              "--api=containers",
1434              "--project-uuid", project_uuid,
1435              "--name", "testing 123",
1436              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1437             capture_stdout, sys.stderr, api_client=stubs.api)
1438         self.assertEqual(exited, 0)
1439
1440         stubs.api.pipeline_templates().create.refute_called()
1441         stubs.api.container_requests().create.refute_called()
1442
1443         body = {
1444             "workflow": {
1445                 "owner_uuid": project_uuid,
1446                 "name": "testing 123",
1447                 "description": "",
1448                 "definition": self.expect_workflow,
1449             }
1450         }
1451         stubs.api.workflows().create.assert_called_with(
1452             body=JsonDiffMatcher(body))
1453
1454         self.assertEqual(capture_stdout.getvalue(),
1455                          stubs.expect_workflow_uuid + '\n')
1456
1457     @stubs
1458     def test_incompatible_api(self, stubs):
1459         capture_stderr = cStringIO.StringIO()
1460         logging.getLogger('arvados.cwl-runner').addHandler(
1461             logging.StreamHandler(capture_stderr))
1462
1463         exited = arvados_cwl.main(
1464             ["--update-workflow", self.existing_workflow_uuid,
1465              "--api=jobs",
1466              "--debug",
1467              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1468             sys.stderr, sys.stderr, api_client=stubs.api)
1469         self.assertEqual(exited, 1)
1470         self.assertRegexpMatches(
1471             capture_stderr.getvalue(),
1472             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1473
1474     @stubs
1475     def test_update(self, stubs):
1476         capture_stdout = cStringIO.StringIO()
1477
1478         exited = arvados_cwl.main(
1479             ["--update-workflow", self.existing_workflow_uuid,
1480              "--debug",
1481              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1482             capture_stdout, sys.stderr, api_client=stubs.api)
1483         self.assertEqual(exited, 0)
1484
1485         body = {
1486             "workflow": {
1487                 "name": "submit_wf.cwl",
1488                 "description": "",
1489                 "definition": self.expect_workflow,
1490             }
1491         }
1492         stubs.api.workflows().update.assert_called_with(
1493             uuid=self.existing_workflow_uuid,
1494             body=JsonDiffMatcher(body))
1495         self.assertEqual(capture_stdout.getvalue(),
1496                          self.existing_workflow_uuid + '\n')
1497
1498
1499     @stubs
1500     def test_update_name(self, stubs):
1501         capture_stdout = cStringIO.StringIO()
1502
1503         exited = arvados_cwl.main(
1504             ["--update-workflow", self.existing_workflow_uuid,
1505              "--debug", "--name", "testing 123",
1506              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1507             capture_stdout, sys.stderr, api_client=stubs.api)
1508         self.assertEqual(exited, 0)
1509
1510         body = {
1511             "workflow": {
1512                 "name": "testing 123",
1513                 "description": "",
1514                 "definition": self.expect_workflow,
1515             }
1516         }
1517         stubs.api.workflows().update.assert_called_with(
1518             uuid=self.existing_workflow_uuid,
1519             body=JsonDiffMatcher(body))
1520         self.assertEqual(capture_stdout.getvalue(),
1521                          self.existing_workflow_uuid + '\n')
1522
1523
1524     @stubs
1525     def test_create_collection_per_tool(self, stubs):
1526         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1527
1528         capture_stdout = cStringIO.StringIO()
1529
1530         exited = arvados_cwl.main(
1531             ["--create-workflow", "--debug",
1532              "--api=containers",
1533              "--project-uuid", project_uuid,
1534              "tests/collection_per_tool/collection_per_tool.cwl"],
1535             capture_stdout, sys.stderr, api_client=stubs.api)
1536         self.assertEqual(exited, 0)
1537
1538         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1539         expect_workflow = StripYAMLComments(open(toolfile).read())
1540
1541         body = {
1542             "workflow": {
1543                 "owner_uuid": project_uuid,
1544                 "name": "collection_per_tool.cwl",
1545                 "description": "",
1546                 "definition": expect_workflow,
1547             }
1548         }
1549         stubs.api.workflows().create.assert_called_with(
1550             body=JsonDiffMatcher(body))
1551
1552         self.assertEqual(capture_stdout.getvalue(),
1553                          stubs.expect_workflow_uuid + '\n')
1554
1555 class TestTemplateInputs(unittest.TestCase):
1556     expect_template = {
1557         "components": {
1558             "inputs_test.cwl": {
1559                 'runtime_constraints': {
1560                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1561                     'min_ram_mb_per_node': 1024
1562                 },
1563                 'script_parameters': {
1564                     'cwl:tool':
1565                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1566                     'optionalFloatInput': None,
1567                     'fileInput': {
1568                         'type': 'File',
1569                         'dataclass': 'File',
1570                         'required': True,
1571                         'title': "It's a file; we expect to find some characters in it.",
1572                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1573                     },
1574                     'floatInput': {
1575                         'type': 'float',
1576                         'dataclass': 'number',
1577                         'required': True,
1578                         'title': 'Floats like a duck',
1579                         'default': 0.1,
1580                         'value': 0.1,
1581                     },
1582                     'optionalFloatInput': {
1583                         'type': ['null', 'float'],
1584                         'dataclass': 'number',
1585                         'required': False,
1586                     },
1587                     'boolInput': {
1588                         'type': 'boolean',
1589                         'dataclass': 'boolean',
1590                         'required': True,
1591                         'title': 'True or false?',
1592                     },
1593                 },
1594                 'repository': 'arvados',
1595                 'script_version': 'master',
1596                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1597                 'script': 'cwl-runner',
1598             },
1599         },
1600         "name": "inputs_test.cwl",
1601     }
1602
1603     @stubs
1604     def test_inputs_empty(self, stubs):
1605         exited = arvados_cwl.main(
1606             ["--create-template",
1607              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1608             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1609         self.assertEqual(exited, 0)
1610
1611         stubs.api.pipeline_templates().create.assert_called_with(
1612             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1613
1614     @stubs
1615     def test_inputs(self, stubs):
1616         exited = arvados_cwl.main(
1617             ["--create-template",
1618              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1619             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1620         self.assertEqual(exited, 0)
1621
1622         expect_template = copy.deepcopy(self.expect_template)
1623         params = expect_template[
1624             "components"]["inputs_test.cwl"]["script_parameters"]
1625         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1626         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1627         params["floatInput"]["value"] = 1.234
1628         params["boolInput"]["value"] = True
1629
1630         stubs.api.pipeline_templates().create.assert_called_with(
1631             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)