sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:debug': True,
 172                         'arv:enable_reuse': True,
 173                         'arv:on_error': 'continue'
 174                     },
 175                     'repository': 'arvados',
 176                     'script_version': 'master',
 177                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 178                     'script': 'cwl-runner',
 179                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 180                 }
 181             }
 182         }
 183         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 184         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 185         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 186         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 187         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 188             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 189             "state": "Queued"
 190         }
 191         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 192         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 193
 194         with open("tests/wf/submit_wf_packed.cwl") as f:
 195             expect_packed_workflow = yaml.round_trip_load(f)
 196
 197         stubs.expect_container_spec = {
 198             'priority': 500,
 199             'mounts': {
 200                 '/var/spool/cwl': {
 201                     'writable': True,
 202                     'kind': 'collection'
 203                 },
 204                 '/var/lib/cwl/workflow.json': {
 205                     'content': expect_packed_workflow,
 206                     'kind': 'json'
 207                 },
 208                 'stdout': {
 209                     'path': '/var/spool/cwl/cwl.output.json',
 210                     'kind': 'file'
 211                 },
 212                 '/var/lib/cwl/cwl.input.json': {
 213                     'kind': 'json',
 214                     'content': {
 215                         'y': {
 216                             'basename': '99999999999999999999999999999998+99',
 217                             'location': 'keep:99999999999999999999999999999998+99',
 218                             'class': 'Directory'},
 219                         'x': {
 220                             'basename': u'blorp.txt',
 221                             'class': 'File',
 222                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 223                             "size": 16
 224                         },
 225                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 226                             {'basename': 'renamed.txt',
 227                              'class': 'File',
 228                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 229                             }
 230                         ]}
 231                     },
 232                     'kind': 'json'
 233                 }
 234             },
 235             'secret_mounts': {},
 236             'state': 'Committed',
 237             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 238                         '--no-log-timestamps', '--disable-validate',
 239                         '--eval-timeout=20', '--thread-count=4',
 240                         '--enable-reuse', '--debug', '--on-error=continue',
 241                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 242             'name': 'submit_wf.cwl',
 243             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 244             'output_path': '/var/spool/cwl',
 245             'cwd': '/var/spool/cwl',
 246             'runtime_constraints': {
 247                 'API': True,
 248                 'vcpus': 1,
 249                 'ram': 1024*1024*1024
 250             },
 251             'use_existing': True,
 252             'properties': {},
 253             'secret_mounts': {}
 254         }
 255
 256         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 257         stubs.api.workflows().create().execute.return_value = {
 258             "uuid": stubs.expect_workflow_uuid,
 259         }
 260         def update_mock(**kwargs):
 261             stubs.updated_uuid = kwargs.get('uuid')
 262             return mock.DEFAULT
 263         stubs.api.workflows().update.side_effect = update_mock
 264         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 265             "uuid": stubs.updated_uuid,
 266         }
 267
 268         return func(self, stubs, *args, **kwargs)
 269     return wrapped
 270
 271
 272 class TestSubmit(unittest.TestCase):
 273     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 274     @mock.patch("time.sleep")
 275     @stubs
 276     def test_submit(self, stubs, tm, arvdock):
 277         capture_stdout = cStringIO.StringIO()
 278         exited = arvados_cwl.main(
 279             ["--submit", "--no-wait", "--api=jobs", "--debug",
 280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 281             capture_stdout, sys.stderr, api_client=stubs.api)
 282         self.assertEqual(exited, 0)
 283
 284         stubs.api.collections().create.assert_has_calls([
 285             mock.call(body=JsonDiffMatcher({
 286                 'manifest_text':
 287                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 288                 'replication_desired': None,
 289                 'name': 'submit_tool.cwl dependencies',
 290             }), ensure_unique_name=True),
 291             mock.call(body=JsonDiffMatcher({
 292                 'manifest_text':
 293                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 294                 'replication_desired': None,
 295                 'name': 'submit_wf.cwl input',
 296             }), ensure_unique_name=True),
 297             mock.call(body=JsonDiffMatcher({
 298                 'manifest_text':
 299                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 300                 'replication_desired': None,
 301                 'name': 'submit_wf.cwl',
 302             }), ensure_unique_name=True)        ])
 303
 304         arvdock.assert_has_calls([
 305             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 306             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 307         ])
 308
 309         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 310         stubs.api.pipeline_instances().create.assert_called_with(
 311             body=JsonDiffMatcher(expect_pipeline))
 312         self.assertEqual(capture_stdout.getvalue(),
 313                          stubs.expect_pipeline_uuid + '\n')
 314
 315
 316     @mock.patch("time.sleep")
 317     @stubs
 318     def test_submit_no_reuse(self, stubs, tm):
 319         capture_stdout = cStringIO.StringIO()
 320         exited = arvados_cwl.main(
 321             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 322              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 323             capture_stdout, sys.stderr, api_client=stubs.api)
 324         self.assertEqual(exited, 0)
 325
 326         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 327         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 328         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 329
 330         stubs.api.pipeline_instances().create.assert_called_with(
 331             body=JsonDiffMatcher(expect_pipeline))
 332         self.assertEqual(capture_stdout.getvalue(),
 333                          stubs.expect_pipeline_uuid + '\n')
 334
 335     @mock.patch("time.sleep")
 336     @stubs
 337     def test_submit_on_error(self, stubs, tm):
 338         capture_stdout = cStringIO.StringIO()
 339         exited = arvados_cwl.main(
 340             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 341              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 342             capture_stdout, sys.stderr, api_client=stubs.api)
 343         self.assertEqual(exited, 0)
 344
 345         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 346         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 347
 348         stubs.api.pipeline_instances().create.assert_called_with(
 349             body=JsonDiffMatcher(expect_pipeline))
 350         self.assertEqual(capture_stdout.getvalue(),
 351                          stubs.expect_pipeline_uuid + '\n')
 352
 353
 354     @mock.patch("time.sleep")
 355     @stubs
 356     def test_submit_runner_ram(self, stubs, tm):
 357         capture_stdout = cStringIO.StringIO()
 358         exited = arvados_cwl.main(
 359             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 360              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 361             capture_stdout, sys.stderr, api_client=stubs.api)
 362         self.assertEqual(exited, 0)
 363
 364         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 365         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 366
 367         stubs.api.pipeline_instances().create.assert_called_with(
 368             body=JsonDiffMatcher(expect_pipeline))
 369         self.assertEqual(capture_stdout.getvalue(),
 370                          stubs.expect_pipeline_uuid + '\n')
 371
 372
 373     @mock.patch("time.sleep")
 374     @stubs
 375     def test_submit_invalid_runner_ram(self, stubs, tm):
 376         capture_stdout = cStringIO.StringIO()
 377         exited = arvados_cwl.main(
 378             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 379              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 380             capture_stdout, sys.stderr, api_client=stubs.api)
 381         self.assertEqual(exited, 1)
 382
 383     @mock.patch("time.sleep")
 384     @stubs
 385     def test_submit_output_name(self, stubs, tm):
 386         output_name = "test_output_name"
 387
 388         capture_stdout = cStringIO.StringIO()
 389         exited = arvados_cwl.main(
 390             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 391              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 392             capture_stdout, sys.stderr, api_client=stubs.api)
 393         self.assertEqual(exited, 0)
 394
 395         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 396         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 397
 398         stubs.api.pipeline_instances().create.assert_called_with(
 399             body=JsonDiffMatcher(expect_pipeline))
 400         self.assertEqual(capture_stdout.getvalue(),
 401                          stubs.expect_pipeline_uuid + '\n')
 402
 403
 404     @mock.patch("time.sleep")
 405     @stubs
 406     def test_submit_pipeline_name(self, stubs, tm):
 407         capture_stdout = cStringIO.StringIO()
 408         exited = arvados_cwl.main(
 409             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 410              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411             capture_stdout, sys.stderr, api_client=stubs.api)
 412         self.assertEqual(exited, 0)
 413
 414         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 415         expect_pipeline["name"] = "hello job 123"
 416
 417         stubs.api.pipeline_instances().create.assert_called_with(
 418             body=JsonDiffMatcher(expect_pipeline))
 419         self.assertEqual(capture_stdout.getvalue(),
 420                          stubs.expect_pipeline_uuid + '\n')
 421
 422     @mock.patch("time.sleep")
 423     @stubs
 424     def test_submit_output_tags(self, stubs, tm):
 425         output_tags = "tag0,tag1,tag2"
 426
 427         capture_stdout = cStringIO.StringIO()
 428         exited = arvados_cwl.main(
 429             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 430              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 431             capture_stdout, sys.stderr, api_client=stubs.api)
 432         self.assertEqual(exited, 0)
 433
 434         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 435         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 436
 437         stubs.api.pipeline_instances().create.assert_called_with(
 438             body=JsonDiffMatcher(expect_pipeline))
 439         self.assertEqual(capture_stdout.getvalue(),
 440                          stubs.expect_pipeline_uuid + '\n')
 441
 442     @mock.patch("time.sleep")
 443     @stubs
 444     def test_submit_with_project_uuid(self, stubs, tm):
 445         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 446
 447         exited = arvados_cwl.main(
 448             ["--submit", "--no-wait", "--debug",
 449              "--project-uuid", project_uuid,
 450              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 451             sys.stdout, sys.stderr, api_client=stubs.api)
 452         self.assertEqual(exited, 0)
 453
 454         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 455         expect_pipeline["owner_uuid"] = project_uuid
 456         stubs.api.pipeline_instances().create.assert_called_with(
 457             body=JsonDiffMatcher(expect_pipeline))
 458
 459     @stubs
 460     def test_submit_container(self, stubs):
 461         capture_stdout = cStringIO.StringIO()
 462         try:
 463             exited = arvados_cwl.main(
 464                 ["--submit", "--no-wait", "--api=containers", "--debug",
 465                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 466                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 467             self.assertEqual(exited, 0)
 468         except:
 469             logging.exception("")
 470
 471         stubs.api.collections().create.assert_has_calls([
 472             mock.call(body=JsonDiffMatcher({
 473                 'manifest_text':
 474                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 475                 'replication_desired': None,
 476                 'name': 'submit_tool.cwl dependencies',
 477             }), ensure_unique_name=True),
 478             mock.call(body=JsonDiffMatcher({
 479                 'manifest_text':
 480                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 481                 'replication_desired': None,
 482                 'name': 'submit_wf.cwl input',
 483             }), ensure_unique_name=True)])
 484
 485         expect_container = copy.deepcopy(stubs.expect_container_spec)
 486         stubs.api.container_requests().create.assert_called_with(
 487             body=JsonDiffMatcher(expect_container))
 488         self.assertEqual(capture_stdout.getvalue(),
 489                          stubs.expect_container_request_uuid + '\n')
 490
 491     @stubs
 492     def test_submit_container_no_reuse(self, stubs):
 493         capture_stdout = cStringIO.StringIO()
 494         try:
 495             exited = arvados_cwl.main(
 496                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 497                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 498                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 499             self.assertEqual(exited, 0)
 500         except:
 501             logging.exception("")
 502
 503         expect_container = copy.deepcopy(stubs.expect_container_spec)
 504         expect_container["command"] = [
 505             'arvados-cwl-runner', '--local', '--api=containers',
 506             '--no-log-timestamps', '--disable-validate',
 507             '--eval-timeout=20', '--thread-count=4',
 508             '--disable-reuse', '--debug', '--on-error=continue',
 509             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 510         expect_container["use_existing"] = False
 511
 512         stubs.api.container_requests().create.assert_called_with(
 513             body=JsonDiffMatcher(expect_container))
 514         self.assertEqual(capture_stdout.getvalue(),
 515                          stubs.expect_container_request_uuid + '\n')
 516
 517
 518     @stubs
 519     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 520         capture_stdout = cStringIO.StringIO()
 521
 522         exited = arvados_cwl.main(
 523             ["--submit", "--no-wait", "--api=containers", "--debug",
 524              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 525             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 526         self.assertEqual(exited, 0)
 527
 528         expect_container = copy.deepcopy(stubs.expect_container_spec)
 529         expect_container["command"] = [
 530             'arvados-cwl-runner', '--local', '--api=containers',
 531             '--no-log-timestamps', '--disable-validate',
 532             '--eval-timeout=20', '--thread-count=4',
 533             '--disable-reuse', '--debug', '--on-error=continue',
 534             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 535         expect_container["use_existing"] = False
 536         expect_container["name"] = "submit_wf_no_reuse.cwl"
 537         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 538             {
 539                 "class": "http://arvados.org/cwl#ReuseRequirement",
 540                 "enableReuse": False,
 541             },
 542         ]
 543         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 544             "arv": "http://arvados.org/cwl#",
 545             "cwltool": "http://commonwl.org/cwltool#"
 546         }
 547
 548         stubs.api.container_requests().create.assert_called_with(
 549             body=JsonDiffMatcher(expect_container))
 550         self.assertEqual(capture_stdout.getvalue(),
 551                          stubs.expect_container_request_uuid + '\n')
 552
 553
 554     @stubs
 555     def test_submit_container_on_error(self, stubs):
 556         capture_stdout = cStringIO.StringIO()
 557         try:
 558             exited = arvados_cwl.main(
 559                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 560                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 561                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 562             self.assertEqual(exited, 0)
 563         except:
 564             logging.exception("")
 565
 566         expect_container = copy.deepcopy(stubs.expect_container_spec)
 567         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 568                                        '--no-log-timestamps', '--disable-validate',
 569                                        '--eval-timeout=20', '--thread-count=4',
 570                                        '--enable-reuse', '--debug', '--on-error=stop',
 571                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 572
 573         stubs.api.container_requests().create.assert_called_with(
 574             body=JsonDiffMatcher(expect_container))
 575         self.assertEqual(capture_stdout.getvalue(),
 576                          stubs.expect_container_request_uuid + '\n')
 577
 578     @stubs
 579     def test_submit_container_output_name(self, stubs):
 580         output_name = "test_output_name"
 581
 582         capture_stdout = cStringIO.StringIO()
 583         try:
 584             exited = arvados_cwl.main(
 585                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 586                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 587                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 588             self.assertEqual(exited, 0)
 589         except:
 590             logging.exception("")
 591
 592         expect_container = copy.deepcopy(stubs.expect_container_spec)
 593         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 594                                        '--no-log-timestamps', '--disable-validate',
 595                                        '--eval-timeout=20', '--thread-count=4',
 596                                        '--enable-reuse',
 597                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 598                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 599         expect_container["output_name"] = output_name
 600
 601         stubs.api.container_requests().create.assert_called_with(
 602             body=JsonDiffMatcher(expect_container))
 603         self.assertEqual(capture_stdout.getvalue(),
 604                          stubs.expect_container_request_uuid + '\n')
 605
 606
 607     @stubs
 608     def test_submit_container_output_ttl(self, stubs):
 609         capture_stdout = cStringIO.StringIO()
 610         try:
 611             exited = arvados_cwl.main(
 612                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 613                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 614                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 615             self.assertEqual(exited, 0)
 616         except:
 617             logging.exception("")
 618
 619         expect_container = copy.deepcopy(stubs.expect_container_spec)
 620         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 621                                        '--no-log-timestamps', '--disable-validate',
 622                                        '--eval-timeout=20', '--thread-count=4',
 623                                        '--enable-reuse', '--debug', '--on-error=continue',
 624                                        "--intermediate-output-ttl=3600",
 625                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 626
 627         stubs.api.container_requests().create.assert_called_with(
 628             body=JsonDiffMatcher(expect_container))
 629         self.assertEqual(capture_stdout.getvalue(),
 630                          stubs.expect_container_request_uuid + '\n')
 631
 632     @stubs
 633     def test_submit_container_trash_intermediate(self, stubs):
 634         capture_stdout = cStringIO.StringIO()
 635         try:
 636             exited = arvados_cwl.main(
 637                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 638                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 639                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 640             self.assertEqual(exited, 0)
 641         except:
 642             logging.exception("")
 643
 644         expect_container = copy.deepcopy(stubs.expect_container_spec)
 645         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 646                                        '--no-log-timestamps', '--disable-validate',
 647                                        '--eval-timeout=20', '--thread-count=4',
 648                                        '--enable-reuse', '--debug', '--on-error=continue',
 649                                        "--trash-intermediate",
 650                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 651
 652         stubs.api.container_requests().create.assert_called_with(
 653             body=JsonDiffMatcher(expect_container))
 654         self.assertEqual(capture_stdout.getvalue(),
 655                          stubs.expect_container_request_uuid + '\n')
 656
 657     @stubs
 658     def test_submit_container_output_tags(self, stubs):
 659         output_tags = "tag0,tag1,tag2"
 660
 661         capture_stdout = cStringIO.StringIO()
 662         try:
 663             exited = arvados_cwl.main(
 664                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 665                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 666                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 667             self.assertEqual(exited, 0)
 668         except:
 669             logging.exception("")
 670
 671         expect_container = copy.deepcopy(stubs.expect_container_spec)
 672         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 673                                        '--no-log-timestamps', '--disable-validate',
 674                                        '--eval-timeout=20', '--thread-count=4',
 675                                        '--enable-reuse',
 676                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 677                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 678
 679         stubs.api.container_requests().create.assert_called_with(
 680             body=JsonDiffMatcher(expect_container))
 681         self.assertEqual(capture_stdout.getvalue(),
 682                          stubs.expect_container_request_uuid + '\n')
 683
 684     @stubs
 685     def test_submit_container_runner_ram(self, stubs):
 686         capture_stdout = cStringIO.StringIO()
 687         try:
 688             exited = arvados_cwl.main(
 689                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 690                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 691                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 692             self.assertEqual(exited, 0)
 693         except:
 694             logging.exception("")
 695
 696         expect_container = copy.deepcopy(stubs.expect_container_spec)
 697         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 698
 699         stubs.api.container_requests().create.assert_called_with(
 700             body=JsonDiffMatcher(expect_container))
 701         self.assertEqual(capture_stdout.getvalue(),
 702                          stubs.expect_container_request_uuid + '\n')
 703
 704     @mock.patch("arvados.collection.CollectionReader")
 705     @mock.patch("time.sleep")
 706     @stubs
 707     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 708         capture_stdout = cStringIO.StringIO()
 709         exited = arvados_cwl.main(
 710             ["--submit", "--no-wait", "--api=containers", "--debug",
 711              "tests/wf/submit_keepref_wf.cwl"],
 712             capture_stdout, sys.stderr, api_client=stubs.api)
 713         self.assertEqual(exited, 0)
 714
 715
 716     @mock.patch("arvados.collection.CollectionReader")
 717     @mock.patch("time.sleep")
 718     @stubs
 719     def test_submit_keepref(self, stubs, tm, reader):
 720         capture_stdout = cStringIO.StringIO()
 721
 722         with open("tests/wf/expect_arvworkflow.cwl") as f:
 723             reader().open().__enter__().read.return_value = f.read()
 724
 725         exited = arvados_cwl.main(
 726             ["--submit", "--no-wait", "--api=containers", "--debug",
 727              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 728             capture_stdout, sys.stderr, api_client=stubs.api)
 729         self.assertEqual(exited, 0)
 730
 731         expect_container = {
 732             'priority': 500,
 733             'mounts': {
 734                 '/var/spool/cwl': {
 735                     'writable': True,
 736                     'kind': 'collection'
 737                 },
 738                 'stdout': {
 739                     'path': '/var/spool/cwl/cwl.output.json',
 740                     'kind': 'file'
 741                 },
 742                 '/var/lib/cwl/workflow': {
 743                     'portable_data_hash': '99999999999999999999999999999994+99',
 744                     'kind': 'collection'
 745                 },
 746                 '/var/lib/cwl/cwl.input.json': {
 747                     'content': {
 748                         'x': 'XxX'
 749                     },
 750                     'kind': 'json'
 751                 }
 752             }, 'state': 'Committed',
 753             'output_path': '/var/spool/cwl',
 754             'name': 'expect_arvworkflow.cwl#main',
 755             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 756             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 757                         '--no-log-timestamps', '--disable-validate',
 758                         '--eval-timeout=20', '--thread-count=4',
 759                         '--enable-reuse', '--debug', '--on-error=continue',
 760                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 761             'cwd': '/var/spool/cwl',
 762             'runtime_constraints': {
 763                 'API': True,
 764                 'vcpus': 1,
 765                 'ram': 1073741824
 766             },
 767             'use_existing': True,
 768             'properties': {},
 769             'secret_mounts': {}
 770         }
 771
 772         stubs.api.container_requests().create.assert_called_with(
 773             body=JsonDiffMatcher(expect_container))
 774         self.assertEqual(capture_stdout.getvalue(),
 775                          stubs.expect_container_request_uuid + '\n')
 776
 777
 778     @mock.patch("arvados.collection.CollectionReader")
 779     @mock.patch("time.sleep")
 780     @stubs
 781     def test_submit_jobs_keepref(self, stubs, tm, reader):
 782         capture_stdout = cStringIO.StringIO()
 783
 784         with open("tests/wf/expect_arvworkflow.cwl") as f:
 785             reader().open().__enter__().read.return_value = f.read()
 786
 787         exited = arvados_cwl.main(
 788             ["--submit", "--no-wait", "--api=jobs", "--debug",
 789              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 790             capture_stdout, sys.stderr, api_client=stubs.api)
 791         self.assertEqual(exited, 0)
 792
 793         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 794         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 795         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 796         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 797         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 798         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 799         stubs.api.pipeline_instances().create.assert_called_with(
 800             body=JsonDiffMatcher(expect_pipeline))
 801
 802     @mock.patch("time.sleep")
 803     @stubs
 804     def test_submit_arvworkflow(self, stubs, tm):
 805         capture_stdout = cStringIO.StringIO()
 806
 807         with open("tests/wf/expect_arvworkflow.cwl") as f:
 808             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 809
 810         exited = arvados_cwl.main(
 811             ["--submit", "--no-wait", "--api=containers", "--debug",
 812              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 813             capture_stdout, sys.stderr, api_client=stubs.api)
 814         self.assertEqual(exited, 0)
 815
 816         expect_container = {
 817             'priority': 500,
 818             'mounts': {
 819                 '/var/spool/cwl': {
 820                     'writable': True,
 821                     'kind': 'collection'
 822                 },
 823                 'stdout': {
 824                     'path': '/var/spool/cwl/cwl.output.json',
 825                     'kind': 'file'
 826                 },
 827                 '/var/lib/cwl/workflow.json': {
 828                     'kind': 'json',
 829                     'content': {
 830                         'cwlVersion': 'v1.0',
 831                         '$graph': [
 832                             {
 833                                 'id': '#main',
 834                                 'inputs': [
 835                                     {'type': 'string', 'id': '#main/x'}
 836                                 ],
 837                                 'steps': [
 838                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 839                                      'run': '#submit_tool.cwl',
 840                                      'id': '#main/step1',
 841                                      'out': []}
 842                                 ],
 843                                 'class': 'Workflow',
 844                                 'outputs': []
 845                             },
 846                             {
 847                                 'inputs': [
 848                                     {
 849                                         'inputBinding': {'position': 1},
 850                                         'type': 'string',
 851                                         'id': '#submit_tool.cwl/x'}
 852                                 ],
 853                                 'requirements': [
 854                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 855                                 ],
 856                                 'id': '#submit_tool.cwl',
 857                                 'outputs': [],
 858                                 'baseCommand': 'cat',
 859                                 'class': 'CommandLineTool'
 860                             }
 861                         ]
 862                     }
 863                 },
 864                 '/var/lib/cwl/cwl.input.json': {
 865                     'content': {
 866                         'x': 'XxX'
 867                     },
 868                     'kind': 'json'
 869                 }
 870             }, 'state': 'Committed',
 871             'output_path': '/var/spool/cwl',
 872             'name': 'a test workflow',
 873             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 874             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 875                         '--no-log-timestamps', '--disable-validate',
 876                         '--eval-timeout=20', '--thread-count=4',
 877                         '--enable-reuse', '--debug', '--on-error=continue',
 878                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 879             'cwd': '/var/spool/cwl',
 880             'runtime_constraints': {
 881                 'API': True,
 882                 'vcpus': 1,
 883                 'ram': 1073741824
 884             },
 885             'use_existing': True,
 886             'properties': {
 887                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 888             },
 889             'secret_mounts': {}
 890         }
 891
 892         stubs.api.container_requests().create.assert_called_with(
 893             body=JsonDiffMatcher(expect_container))
 894         self.assertEqual(capture_stdout.getvalue(),
 895                          stubs.expect_container_request_uuid + '\n')
 896
 897
 898     @stubs
 899     def test_submit_container_name(self, stubs):
 900         capture_stdout = cStringIO.StringIO()
 901         try:
 902             exited = arvados_cwl.main(
 903                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 904                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 905                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 906             self.assertEqual(exited, 0)
 907         except:
 908             logging.exception("")
 909
 910         expect_container = copy.deepcopy(stubs.expect_container_spec)
 911         expect_container["name"] = "hello container 123"
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917
 918
 919     @stubs
 920     def test_submit_container_project(self, stubs):
 921         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 922         capture_stdout = cStringIO.StringIO()
 923         try:
 924             exited = arvados_cwl.main(
 925                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 926                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 927                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 928             self.assertEqual(exited, 0)
 929         except:
 930             logging.exception("")
 931
 932         expect_container = copy.deepcopy(stubs.expect_container_spec)
 933         expect_container["owner_uuid"] = project_uuid
 934         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 935                                        '--no-log-timestamps', '--disable-validate',
 936                                        "--eval-timeout=20", "--thread-count=4",
 937                                        '--enable-reuse', '--debug', '--on-error=continue',
 938                                        '--project-uuid='+project_uuid,
 939                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 940
 941         stubs.api.container_requests().create.assert_called_with(
 942             body=JsonDiffMatcher(expect_container))
 943         self.assertEqual(capture_stdout.getvalue(),
 944                          stubs.expect_container_request_uuid + '\n')
 945
 946     @stubs
 947     def test_submit_container_eval_timeout(self, stubs):
 948         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 949         capture_stdout = cStringIO.StringIO()
 950         try:
 951             exited = arvados_cwl.main(
 952                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 953                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 954                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 955             self.assertEqual(exited, 0)
 956         except:
 957             logging.exception("")
 958
 959         expect_container = copy.deepcopy(stubs.expect_container_spec)
 960         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 961                                        '--no-log-timestamps', '--disable-validate',
 962                                        '--eval-timeout=60.0', '--thread-count=4',
 963                                        '--enable-reuse', '--debug', '--on-error=continue',
 964                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 965
 966         stubs.api.container_requests().create.assert_called_with(
 967             body=JsonDiffMatcher(expect_container))
 968         self.assertEqual(capture_stdout.getvalue(),
 969                          stubs.expect_container_request_uuid + '\n')
 970
 971
 972     @stubs
 973     def test_submit_container_thread_count(self, stubs):
 974         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 975         capture_stdout = cStringIO.StringIO()
 976         try:
 977             exited = arvados_cwl.main(
 978                 ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 979                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 980                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 981             self.assertEqual(exited, 0)
 982         except:
 983             logging.exception("")
 984
 985         expect_container = copy.deepcopy(stubs.expect_container_spec)
 986         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 987                                        '--no-log-timestamps', '--disable-validate',
 988                                        '--eval-timeout=20', '--thread-count=20',
 989                                        '--enable-reuse', '--debug', '--on-error=continue',
 990                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 991
 992         stubs.api.container_requests().create.assert_called_with(
 993             body=JsonDiffMatcher(expect_container))
 994         self.assertEqual(capture_stdout.getvalue(),
 995                          stubs.expect_container_request_uuid + '\n')
 996
 997
 998     @stubs
 999     def test_submit_job_runner_image(self, stubs):
1000         capture_stdout = cStringIO.StringIO()
1001         try:
1002             exited = arvados_cwl.main(
1003                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1004                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1005                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1006             self.assertEqual(exited, 0)
1007         except:
1008             logging.exception("")
1009
1010         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
1011
1012         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1013         stubs.api.pipeline_instances().create.assert_called_with(
1014             body=JsonDiffMatcher(expect_pipeline))
1015         self.assertEqual(capture_stdout.getvalue(),
1016                          stubs.expect_pipeline_uuid + '\n')
1017
1018     @stubs
1019     def test_submit_container_runner_image(self, stubs):
1020         capture_stdout = cStringIO.StringIO()
1021         try:
1022             exited = arvados_cwl.main(
1023                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1024                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1025                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1026             self.assertEqual(exited, 0)
1027         except:
1028             logging.exception("")
1029
1030         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
1031
1032         expect_container = copy.deepcopy(stubs.expect_container_spec)
1033         stubs.api.container_requests().create.assert_called_with(
1034             body=JsonDiffMatcher(expect_container))
1035         self.assertEqual(capture_stdout.getvalue(),
1036                          stubs.expect_container_request_uuid + '\n')
1037
1038     @stubs
1039     def test_submit_priority(self, stubs):
1040         capture_stdout = cStringIO.StringIO()
1041         try:
1042             exited = arvados_cwl.main(
1043                 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1044                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1045                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1046             self.assertEqual(exited, 0)
1047         except:
1048             logging.exception("")
1049
1050         stubs.expect_container_spec["priority"] = 669
1051
1052         expect_container = copy.deepcopy(stubs.expect_container_spec)
1053         stubs.api.container_requests().create.assert_called_with(
1054             body=JsonDiffMatcher(expect_container))
1055         self.assertEqual(capture_stdout.getvalue(),
1056                          stubs.expect_container_request_uuid + '\n')
1057
1058
1059     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1060     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1061     @mock.patch("arvados.api")
1062     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1063         arvrunner = mock.MagicMock()
1064         arvrunner.project_uuid = ""
1065         api.return_value = mock.MagicMock()
1066         arvrunner.api = api.return_value
1067         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1068                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1069                                                                         "link_class": "docker_image_repo+tag",
1070                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1071                                                                         "owner_uuid": "",
1072                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1073                                                             {"items": [{"created_at": "",
1074                                                                         "head_uuid": "",
1075                                                                         "link_class": "docker_image_hash",
1076                                                                         "name": "123456",
1077                                                                         "owner_uuid": "",
1078                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1079         )
1080         find_one_image_hash.return_value = "123456"
1081
1082         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1083                                                                               "owner_uuid": "",
1084                                                                               "manifest_text": "",
1085                                                                               "properties": ""
1086                                                                           }], "items_available": 1, "offset": 0},)
1087         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1088         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1089                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1090
1091     @stubs
1092     def test_submit_secrets(self, stubs):
1093         capture_stdout = cStringIO.StringIO()
1094         try:
1095             exited = arvados_cwl.main(
1096                 ["--submit", "--no-wait", "--api=containers", "--debug",
1097                  "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1098                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1099             self.assertEqual(exited, 0)
1100         except:
1101             logging.exception("")
1102
1103
1104         expect_container = {
1105             "command": [
1106                 "arvados-cwl-runner",
1107                 "--local",
1108                 "--api=containers",
1109                 "--no-log-timestamps",
1110                 "--disable-validate",
1111                 "--eval-timeout=20",
1112                 '--thread-count=4',
1113                 "--enable-reuse",
1114                 '--debug',
1115                 "--on-error=continue",
1116                 "/var/lib/cwl/workflow.json#main",
1117                 "/var/lib/cwl/cwl.input.json"
1118             ],
1119             "container_image": "arvados/jobs:"+arvados_cwl.__version__,
1120             "cwd": "/var/spool/cwl",
1121             "mounts": {
1122                 "/var/lib/cwl/cwl.input.json": {
1123                     "content": {
1124                         "pw": {
1125                             "$include": "/secrets/s0"
1126                         }
1127                     },
1128                     "kind": "json"
1129                 },
1130                 "/var/lib/cwl/workflow.json": {
1131                     "content": {
1132                         "$graph": [
1133                             {
1134                                 "$namespaces": {
1135                                     "cwltool": "http://commonwl.org/cwltool#"
1136                                 },
1137                                 "arguments": [
1138                                     "md5sum",
1139                                     "example.conf"
1140                                 ],
1141                                 "class": "CommandLineTool",
1142                                 "hints": [
1143                                     {
1144                                         "class": "http://commonwl.org/cwltool#Secrets",
1145                                         "secrets": [
1146                                             "#secret_job.cwl/pw"
1147                                         ]
1148                                     }
1149                                 ],
1150                                 "id": "#secret_job.cwl",
1151                                 "inputs": [
1152                                     {
1153                                         "id": "#secret_job.cwl/pw",
1154                                         "type": "string"
1155                                     }
1156                                 ],
1157                                 "outputs": [
1158                                     {
1159                                         "id": "#secret_job.cwl/out",
1160                                         "type": "stdout"
1161                                     }
1162                                 ],
1163                                 "stdout": "hashed_example.txt",
1164                                 "requirements": [
1165                                     {
1166                                         "class": "InitialWorkDirRequirement",
1167                                         "listing": [
1168                                             {
1169                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1170                                                 "entryname": "example.conf"
1171                                             }
1172                                         ]
1173                                     }
1174                                 ]
1175                             },
1176                             {
1177                                 "class": "Workflow",
1178                                 "hints": [
1179                                     {
1180                                         "class": "DockerRequirement",
1181                                         "dockerPull": "debian:8"
1182                                     },
1183                                     {
1184                                         "class": "http://commonwl.org/cwltool#Secrets",
1185                                         "secrets": [
1186                                             "#main/pw"
1187                                         ]
1188                                     }
1189                                 ],
1190                                 "id": "#main",
1191                                 "inputs": [
1192                                     {
1193                                         "id": "#main/pw",
1194                                         "type": "string"
1195                                     }
1196                                 ],
1197                                 "outputs": [
1198                                     {
1199                                         "id": "#main/out",
1200                                         "outputSource": "#main/step1/out",
1201                                         "type": "File"
1202                                     }
1203                                 ],
1204                                 "steps": [
1205                                     {
1206                                         "id": "#main/step1",
1207                                         "in": [
1208                                             {
1209                                                 "id": "#main/step1/pw",
1210                                                 "source": "#main/pw"
1211                                             }
1212                                         ],
1213                                         "out": [
1214                                             "#main/step1/out"
1215                                         ],
1216                                         "run": "#secret_job.cwl"
1217                                     }
1218                                 ]
1219                             }
1220                         ],
1221                         "cwlVersion": "v1.0"
1222                     },
1223                     "kind": "json"
1224                 },
1225                 "/var/spool/cwl": {
1226                     "kind": "collection",
1227                     "writable": True
1228                 },
1229                 "stdout": {
1230                     "kind": "file",
1231                     "path": "/var/spool/cwl/cwl.output.json"
1232                 }
1233             },
1234             "name": "secret_wf.cwl",
1235             "output_path": "/var/spool/cwl",
1236             "priority": 500,
1237             "properties": {},
1238             "runtime_constraints": {
1239                 "API": True,
1240                 "ram": 1073741824,
1241                 "vcpus": 1
1242             },
1243             "secret_mounts": {
1244                 "/secrets/s0": {
1245                     "content": "blorp",
1246                     "kind": "text"
1247                 }
1248             },
1249             "state": "Committed",
1250             "use_existing": True
1251         }
1252
1253         stubs.api.container_requests().create.assert_called_with(
1254             body=JsonDiffMatcher(expect_container))
1255         self.assertEqual(capture_stdout.getvalue(),
1256                          stubs.expect_container_request_uuid + '\n')
1257
1258     @stubs
1259     def test_submit_request_uuid(self, stubs):
1260         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1261
1262         stubs.api.container_requests().update().execute.return_value = {
1263             "uuid": stubs.expect_container_request_uuid,
1264             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1265             "state": "Queued"
1266         }
1267
1268         capture_stdout = cStringIO.StringIO()
1269         try:
1270             exited = arvados_cwl.main(
1271                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1272                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1273                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1274             self.assertEqual(exited, 0)
1275         except:
1276             logging.exception("")
1277
1278         stubs.api.container_requests().update.assert_called_with(
1279             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1280         self.assertEqual(capture_stdout.getvalue(),
1281                          stubs.expect_container_request_uuid + '\n')
1282
1283
1284 class TestCreateTemplate(unittest.TestCase):
1285     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1286
1287     def _adjust_script_params(self, expect_component):
1288         expect_component['script_parameters']['x'] = {
1289             'dataclass': 'File',
1290             'required': True,
1291             'type': 'File',
1292             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1293         }
1294         expect_component['script_parameters']['y'] = {
1295             'dataclass': 'Collection',
1296             'required': True,
1297             'type': 'Directory',
1298             'value': '99999999999999999999999999999998+99',
1299         }
1300         expect_component['script_parameters']['z'] = {
1301             'dataclass': 'Collection',
1302             'required': True,
1303             'type': 'Directory',
1304         }
1305
1306     @stubs
1307     def test_create(self, stubs):
1308         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1309
1310         capture_stdout = cStringIO.StringIO()
1311
1312         exited = arvados_cwl.main(
1313             ["--create-workflow", "--debug",
1314              "--api=jobs",
1315              "--project-uuid", project_uuid,
1316              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1317             capture_stdout, sys.stderr, api_client=stubs.api)
1318         self.assertEqual(exited, 0)
1319
1320         stubs.api.pipeline_instances().create.refute_called()
1321         stubs.api.jobs().create.refute_called()
1322
1323         expect_component = copy.deepcopy(stubs.expect_job_spec)
1324         self._adjust_script_params(expect_component)
1325         expect_template = {
1326             "components": {
1327                 "submit_wf.cwl": expect_component,
1328             },
1329             "name": "submit_wf.cwl",
1330             "owner_uuid": project_uuid,
1331         }
1332         stubs.api.pipeline_templates().create.assert_called_with(
1333             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1334
1335         self.assertEqual(capture_stdout.getvalue(),
1336                          stubs.expect_pipeline_template_uuid + '\n')
1337
1338
1339     @stubs
1340     def test_create_name(self, stubs):
1341         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1342
1343         capture_stdout = cStringIO.StringIO()
1344
1345         exited = arvados_cwl.main(
1346             ["--create-workflow", "--debug",
1347              "--project-uuid", project_uuid,
1348              "--api=jobs",
1349              "--name", "testing 123",
1350              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1351             capture_stdout, sys.stderr, api_client=stubs.api)
1352         self.assertEqual(exited, 0)
1353
1354         stubs.api.pipeline_instances().create.refute_called()
1355         stubs.api.jobs().create.refute_called()
1356
1357         expect_component = copy.deepcopy(stubs.expect_job_spec)
1358         self._adjust_script_params(expect_component)
1359         expect_template = {
1360             "components": {
1361                 "testing 123": expect_component,
1362             },
1363             "name": "testing 123",
1364             "owner_uuid": project_uuid,
1365         }
1366         stubs.api.pipeline_templates().create.assert_called_with(
1367             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1368
1369         self.assertEqual(capture_stdout.getvalue(),
1370                          stubs.expect_pipeline_template_uuid + '\n')
1371
1372
1373     @stubs
1374     def test_update_name(self, stubs):
1375         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1376
1377         capture_stdout = cStringIO.StringIO()
1378
1379         exited = arvados_cwl.main(
1380             ["--update-workflow", self.existing_template_uuid,
1381              "--debug",
1382              "--project-uuid", project_uuid,
1383              "--api=jobs",
1384              "--name", "testing 123",
1385              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1386             capture_stdout, sys.stderr, api_client=stubs.api)
1387         self.assertEqual(exited, 0)
1388
1389         stubs.api.pipeline_instances().create.refute_called()
1390         stubs.api.jobs().create.refute_called()
1391
1392         expect_component = copy.deepcopy(stubs.expect_job_spec)
1393         self._adjust_script_params(expect_component)
1394         expect_template = {
1395             "components": {
1396                 "testing 123": expect_component,
1397             },
1398             "name": "testing 123",
1399             "owner_uuid": project_uuid,
1400         }
1401         stubs.api.pipeline_templates().create.refute_called()
1402         stubs.api.pipeline_templates().update.assert_called_with(
1403             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1404
1405         self.assertEqual(capture_stdout.getvalue(),
1406                          self.existing_template_uuid + '\n')
1407
1408
1409 class TestCreateWorkflow(unittest.TestCase):
1410     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1411     expect_workflow = StripYAMLComments(
1412         open("tests/wf/expect_packed.cwl").read())
1413
1414     @stubs
1415     def test_create(self, stubs):
1416         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1417
1418         capture_stdout = cStringIO.StringIO()
1419
1420         exited = arvados_cwl.main(
1421             ["--create-workflow", "--debug",
1422              "--api=containers",
1423              "--project-uuid", project_uuid,
1424              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1425             capture_stdout, sys.stderr, api_client=stubs.api)
1426         self.assertEqual(exited, 0)
1427
1428         stubs.api.pipeline_templates().create.refute_called()
1429         stubs.api.container_requests().create.refute_called()
1430
1431         body = {
1432             "workflow": {
1433                 "owner_uuid": project_uuid,
1434                 "name": "submit_wf.cwl",
1435                 "description": "",
1436                 "definition": self.expect_workflow,
1437             }
1438         }
1439         stubs.api.workflows().create.assert_called_with(
1440             body=JsonDiffMatcher(body))
1441
1442         self.assertEqual(capture_stdout.getvalue(),
1443                          stubs.expect_workflow_uuid + '\n')
1444
1445
1446     @stubs
1447     def test_create_name(self, stubs):
1448         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1449
1450         capture_stdout = cStringIO.StringIO()
1451
1452         exited = arvados_cwl.main(
1453             ["--create-workflow", "--debug",
1454              "--api=containers",
1455              "--project-uuid", project_uuid,
1456              "--name", "testing 123",
1457              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1458             capture_stdout, sys.stderr, api_client=stubs.api)
1459         self.assertEqual(exited, 0)
1460
1461         stubs.api.pipeline_templates().create.refute_called()
1462         stubs.api.container_requests().create.refute_called()
1463
1464         body = {
1465             "workflow": {
1466                 "owner_uuid": project_uuid,
1467                 "name": "testing 123",
1468                 "description": "",
1469                 "definition": self.expect_workflow,
1470             }
1471         }
1472         stubs.api.workflows().create.assert_called_with(
1473             body=JsonDiffMatcher(body))
1474
1475         self.assertEqual(capture_stdout.getvalue(),
1476                          stubs.expect_workflow_uuid + '\n')
1477
1478     @stubs
1479     def test_incompatible_api(self, stubs):
1480         capture_stderr = cStringIO.StringIO()
1481         logging.getLogger('arvados.cwl-runner').addHandler(
1482             logging.StreamHandler(capture_stderr))
1483
1484         exited = arvados_cwl.main(
1485             ["--update-workflow", self.existing_workflow_uuid,
1486              "--api=jobs",
1487              "--debug",
1488              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1489             sys.stderr, sys.stderr, api_client=stubs.api)
1490         self.assertEqual(exited, 1)
1491         self.assertRegexpMatches(
1492             capture_stderr.getvalue(),
1493             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1494
1495     @stubs
1496     def test_update(self, stubs):
1497         capture_stdout = cStringIO.StringIO()
1498
1499         exited = arvados_cwl.main(
1500             ["--update-workflow", self.existing_workflow_uuid,
1501              "--debug",
1502              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1503             capture_stdout, sys.stderr, api_client=stubs.api)
1504         self.assertEqual(exited, 0)
1505
1506         body = {
1507             "workflow": {
1508                 "name": "submit_wf.cwl",
1509                 "description": "",
1510                 "definition": self.expect_workflow,
1511             }
1512         }
1513         stubs.api.workflows().update.assert_called_with(
1514             uuid=self.existing_workflow_uuid,
1515             body=JsonDiffMatcher(body))
1516         self.assertEqual(capture_stdout.getvalue(),
1517                          self.existing_workflow_uuid + '\n')
1518
1519
1520     @stubs
1521     def test_update_name(self, stubs):
1522         capture_stdout = cStringIO.StringIO()
1523
1524         exited = arvados_cwl.main(
1525             ["--update-workflow", self.existing_workflow_uuid,
1526              "--debug", "--name", "testing 123",
1527              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1528             capture_stdout, sys.stderr, api_client=stubs.api)
1529         self.assertEqual(exited, 0)
1530
1531         body = {
1532             "workflow": {
1533                 "name": "testing 123",
1534                 "description": "",
1535                 "definition": self.expect_workflow,
1536             }
1537         }
1538         stubs.api.workflows().update.assert_called_with(
1539             uuid=self.existing_workflow_uuid,
1540             body=JsonDiffMatcher(body))
1541         self.assertEqual(capture_stdout.getvalue(),
1542                          self.existing_workflow_uuid + '\n')
1543
1544
1545     @stubs
1546     def test_create_collection_per_tool(self, stubs):
1547         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1548
1549         capture_stdout = cStringIO.StringIO()
1550
1551         exited = arvados_cwl.main(
1552             ["--create-workflow", "--debug",
1553              "--api=containers",
1554              "--project-uuid", project_uuid,
1555              "tests/collection_per_tool/collection_per_tool.cwl"],
1556             capture_stdout, sys.stderr, api_client=stubs.api)
1557         self.assertEqual(exited, 0)
1558
1559         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1560         expect_workflow = StripYAMLComments(open(toolfile).read())
1561
1562         body = {
1563             "workflow": {
1564                 "owner_uuid": project_uuid,
1565                 "name": "collection_per_tool.cwl",
1566                 "description": "",
1567                 "definition": expect_workflow,
1568             }
1569         }
1570         stubs.api.workflows().create.assert_called_with(
1571             body=JsonDiffMatcher(body))
1572
1573         self.assertEqual(capture_stdout.getvalue(),
1574                          stubs.expect_workflow_uuid + '\n')
1575
1576 class TestTemplateInputs(unittest.TestCase):
1577     expect_template = {
1578         "components": {
1579             "inputs_test.cwl": {
1580                 'runtime_constraints': {
1581                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1582                     'min_ram_mb_per_node': 1024
1583                 },
1584                 'script_parameters': {
1585                     'cwl:tool':
1586                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1587                     'optionalFloatInput': None,
1588                     'fileInput': {
1589                         'type': 'File',
1590                         'dataclass': 'File',
1591                         'required': True,
1592                         'title': "It's a file; we expect to find some characters in it.",
1593                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1594                     },
1595                     'floatInput': {
1596                         'type': 'float',
1597                         'dataclass': 'number',
1598                         'required': True,
1599                         'title': 'Floats like a duck',
1600                         'default': 0.1,
1601                         'value': 0.1,
1602                     },
1603                     'optionalFloatInput': {
1604                         'type': ['null', 'float'],
1605                         'dataclass': 'number',
1606                         'required': False,
1607                     },
1608                     'boolInput': {
1609                         'type': 'boolean',
1610                         'dataclass': 'boolean',
1611                         'required': True,
1612                         'title': 'True or false?',
1613                     },
1614                 },
1615                 'repository': 'arvados',
1616                 'script_version': 'master',
1617                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1618                 'script': 'cwl-runner',
1619             },
1620         },
1621         "name": "inputs_test.cwl",
1622     }
1623
1624     @stubs
1625     def test_inputs_empty(self, stubs):
1626         exited = arvados_cwl.main(
1627             ["--create-template",
1628              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1629             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1630         self.assertEqual(exited, 0)
1631
1632         stubs.api.pipeline_templates().create.assert_called_with(
1633             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1634
1635     @stubs
1636     def test_inputs(self, stubs):
1637         exited = arvados_cwl.main(
1638             ["--create-template",
1639              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1640             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1641         self.assertEqual(exited, 0)
1642
1643         expect_template = copy.deepcopy(self.expect_template)
1644         params = expect_template[
1645             "components"]["inputs_test.cwl"]["script_parameters"]
1646         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1647         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1648         params["floatInput"]["value"] = 1.234
1649         params["boolInput"]["value"] = True
1650
1651         stubs.api.pipeline_templates().create.assert_called_with(
1652             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)