sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 500,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'secret_mounts': {},
 235             'state': 'Committed',
 236             'owner_uuid': None,
 237             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 238                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 239                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 240             'name': 'submit_wf.cwl',
 241             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 242             'output_path': '/var/spool/cwl',
 243             'cwd': '/var/spool/cwl',
 244             'runtime_constraints': {
 245                 'API': True,
 246                 'vcpus': 1,
 247                 'ram': 1024*1024*1024
 248             },
 249             'use_existing': True,
 250             'properties': {},
 251             'secret_mounts': {}
 252         }
 253
 254         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 255         stubs.api.workflows().create().execute.return_value = {
 256             "uuid": stubs.expect_workflow_uuid,
 257         }
 258         def update_mock(**kwargs):
 259             stubs.updated_uuid = kwargs.get('uuid')
 260             return mock.DEFAULT
 261         stubs.api.workflows().update.side_effect = update_mock
 262         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 263             "uuid": stubs.updated_uuid,
 264         }
 265
 266         return func(self, stubs, *args, **kwargs)
 267     return wrapped
 268
 269
 270 class TestSubmit(unittest.TestCase):
 271     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 272     @mock.patch("time.sleep")
 273     @stubs
 274     def test_submit(self, stubs, tm, arvdock):
 275         capture_stdout = cStringIO.StringIO()
 276         exited = arvados_cwl.main(
 277             ["--submit", "--no-wait", "--api=jobs", "--debug",
 278              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 279             capture_stdout, sys.stderr, api_client=stubs.api)
 280         self.assertEqual(exited, 0)
 281
 282         stubs.api.collections().create.assert_has_calls([
 283             mock.call(body=JsonDiffMatcher({
 284                 'manifest_text':
 285                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 286                 'replication_desired': None,
 287                 'name': 'submit_tool.cwl dependencies',
 288             }), ensure_unique_name=True),
 289             mock.call(body=JsonDiffMatcher({
 290                 'manifest_text':
 291                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 292                 'replication_desired': None,
 293                 'name': 'submit_wf.cwl input',
 294             }), ensure_unique_name=True),
 295             mock.call(body=JsonDiffMatcher({
 296                 'manifest_text':
 297                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 298                 'replication_desired': None,
 299                 'name': 'submit_wf.cwl',
 300             }), ensure_unique_name=True)        ])
 301
 302         arvdock.assert_has_calls([
 303             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 304             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 305         ])
 306
 307         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 308         stubs.api.pipeline_instances().create.assert_called_with(
 309             body=JsonDiffMatcher(expect_pipeline))
 310         self.assertEqual(capture_stdout.getvalue(),
 311                          stubs.expect_pipeline_uuid + '\n')
 312
 313
 314     @mock.patch("time.sleep")
 315     @stubs
 316     def test_submit_no_reuse(self, stubs, tm):
 317         capture_stdout = cStringIO.StringIO()
 318         exited = arvados_cwl.main(
 319             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 320              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 321             capture_stdout, sys.stderr, api_client=stubs.api)
 322         self.assertEqual(exited, 0)
 323
 324         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 325         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 326         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 327
 328         stubs.api.pipeline_instances().create.assert_called_with(
 329             body=JsonDiffMatcher(expect_pipeline))
 330         self.assertEqual(capture_stdout.getvalue(),
 331                          stubs.expect_pipeline_uuid + '\n')
 332
 333     @mock.patch("time.sleep")
 334     @stubs
 335     def test_submit_on_error(self, stubs, tm):
 336         capture_stdout = cStringIO.StringIO()
 337         exited = arvados_cwl.main(
 338             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 339              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 340             capture_stdout, sys.stderr, api_client=stubs.api)
 341         self.assertEqual(exited, 0)
 342
 343         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 344         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 345
 346         stubs.api.pipeline_instances().create.assert_called_with(
 347             body=JsonDiffMatcher(expect_pipeline))
 348         self.assertEqual(capture_stdout.getvalue(),
 349                          stubs.expect_pipeline_uuid + '\n')
 350
 351
 352     @mock.patch("time.sleep")
 353     @stubs
 354     def test_submit_runner_ram(self, stubs, tm):
 355         capture_stdout = cStringIO.StringIO()
 356         exited = arvados_cwl.main(
 357             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 359             capture_stdout, sys.stderr, api_client=stubs.api)
 360         self.assertEqual(exited, 0)
 361
 362         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 363         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 364
 365         stubs.api.pipeline_instances().create.assert_called_with(
 366             body=JsonDiffMatcher(expect_pipeline))
 367         self.assertEqual(capture_stdout.getvalue(),
 368                          stubs.expect_pipeline_uuid + '\n')
 369
 370
 371     @mock.patch("time.sleep")
 372     @stubs
 373     def test_submit_invalid_runner_ram(self, stubs, tm):
 374         capture_stdout = cStringIO.StringIO()
 375         exited = arvados_cwl.main(
 376             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 377              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 378             capture_stdout, sys.stderr, api_client=stubs.api)
 379         self.assertEqual(exited, 1)
 380
 381     @mock.patch("time.sleep")
 382     @stubs
 383     def test_submit_output_name(self, stubs, tm):
 384         output_name = "test_output_name"
 385
 386         capture_stdout = cStringIO.StringIO()
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 389              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 390             capture_stdout, sys.stderr, api_client=stubs.api)
 391         self.assertEqual(exited, 0)
 392
 393         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 394         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 395
 396         stubs.api.pipeline_instances().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_pipeline))
 398         self.assertEqual(capture_stdout.getvalue(),
 399                          stubs.expect_pipeline_uuid + '\n')
 400
 401
 402     @mock.patch("time.sleep")
 403     @stubs
 404     def test_submit_pipeline_name(self, stubs, tm):
 405         capture_stdout = cStringIO.StringIO()
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 408              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 409             capture_stdout, sys.stderr, api_client=stubs.api)
 410         self.assertEqual(exited, 0)
 411
 412         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 413         expect_pipeline["name"] = "hello job 123"
 414
 415         stubs.api.pipeline_instances().create.assert_called_with(
 416             body=JsonDiffMatcher(expect_pipeline))
 417         self.assertEqual(capture_stdout.getvalue(),
 418                          stubs.expect_pipeline_uuid + '\n')
 419
 420     @mock.patch("time.sleep")
 421     @stubs
 422     def test_submit_output_tags(self, stubs, tm):
 423         output_tags = "tag0,tag1,tag2"
 424
 425         capture_stdout = cStringIO.StringIO()
 426         exited = arvados_cwl.main(
 427             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 428              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 429             capture_stdout, sys.stderr, api_client=stubs.api)
 430         self.assertEqual(exited, 0)
 431
 432         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 433         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 434
 435         stubs.api.pipeline_instances().create.assert_called_with(
 436             body=JsonDiffMatcher(expect_pipeline))
 437         self.assertEqual(capture_stdout.getvalue(),
 438                          stubs.expect_pipeline_uuid + '\n')
 439
 440     @mock.patch("time.sleep")
 441     @stubs
 442     def test_submit_with_project_uuid(self, stubs, tm):
 443         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 444
 445         exited = arvados_cwl.main(
 446             ["--submit", "--no-wait",
 447              "--project-uuid", project_uuid,
 448              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 449             sys.stdout, sys.stderr, api_client=stubs.api)
 450         self.assertEqual(exited, 0)
 451
 452         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 453         expect_pipeline["owner_uuid"] = project_uuid
 454         stubs.api.pipeline_instances().create.assert_called_with(
 455             body=JsonDiffMatcher(expect_pipeline))
 456
 457     @stubs
 458     def test_submit_container(self, stubs):
 459         capture_stdout = cStringIO.StringIO()
 460         try:
 461             exited = arvados_cwl.main(
 462                 ["--submit", "--no-wait", "--api=containers", "--debug",
 463                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 464                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 465             self.assertEqual(exited, 0)
 466         except:
 467             logging.exception("")
 468
 469         stubs.api.collections().create.assert_has_calls([
 470             mock.call(body=JsonDiffMatcher({
 471                 'manifest_text':
 472                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 473                 'replication_desired': None,
 474                 'name': 'submit_tool.cwl dependencies',
 475             }), ensure_unique_name=True),
 476             mock.call(body=JsonDiffMatcher({
 477                 'manifest_text':
 478                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 479                 'replication_desired': None,
 480                 'name': 'submit_wf.cwl input',
 481             }), ensure_unique_name=True)])
 482
 483         expect_container = copy.deepcopy(stubs.expect_container_spec)
 484         stubs.api.container_requests().create.assert_called_with(
 485             body=JsonDiffMatcher(expect_container))
 486         self.assertEqual(capture_stdout.getvalue(),
 487                          stubs.expect_container_request_uuid + '\n')
 488
 489     @stubs
 490     def test_submit_container_no_reuse(self, stubs):
 491         capture_stdout = cStringIO.StringIO()
 492         try:
 493             exited = arvados_cwl.main(
 494                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 495                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 496                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 497             self.assertEqual(exited, 0)
 498         except:
 499             logging.exception("")
 500
 501         expect_container = copy.deepcopy(stubs.expect_container_spec)
 502         expect_container["command"] = [
 503             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 504             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 505             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 506         expect_container["use_existing"] = False
 507
 508         stubs.api.container_requests().create.assert_called_with(
 509             body=JsonDiffMatcher(expect_container))
 510         self.assertEqual(capture_stdout.getvalue(),
 511                          stubs.expect_container_request_uuid + '\n')
 512
 513
 514     @stubs
 515     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 516         capture_stdout = cStringIO.StringIO()
 517
 518         exited = arvados_cwl.main(
 519             ["--submit", "--no-wait", "--api=containers", "--debug",
 520              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 521             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 522         self.assertEqual(exited, 0)
 523
 524         expect_container = copy.deepcopy(stubs.expect_container_spec)
 525         expect_container["command"] = [
 526             'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 527             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 528             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 529         expect_container["use_existing"] = False
 530         expect_container["name"] = "submit_wf_no_reuse.cwl"
 531         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 532             {
 533                 "class": "http://arvados.org/cwl#ReuseRequirement",
 534                 "enableReuse": False,
 535             },
 536         ]
 537         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 538             "arv": "http://arvados.org/cwl#",
 539             "cwltool": "http://commonwl.org/cwltool#"
 540         }
 541
 542         stubs.api.container_requests().create.assert_called_with(
 543             body=JsonDiffMatcher(expect_container))
 544         self.assertEqual(capture_stdout.getvalue(),
 545                          stubs.expect_container_request_uuid + '\n')
 546
 547
 548     @stubs
 549     def test_submit_container_on_error(self, stubs):
 550         capture_stdout = cStringIO.StringIO()
 551         try:
 552             exited = arvados_cwl.main(
 553                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 554                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 555                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 556             self.assertEqual(exited, 0)
 557         except:
 558             logging.exception("")
 559
 560         expect_container = copy.deepcopy(stubs.expect_container_spec)
 561         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 562                                                   '--enable-reuse', '--on-error=stop', '--eval-timeout=20',
 563                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 564
 565         stubs.api.container_requests().create.assert_called_with(
 566             body=JsonDiffMatcher(expect_container))
 567         self.assertEqual(capture_stdout.getvalue(),
 568                          stubs.expect_container_request_uuid + '\n')
 569
 570     @stubs
 571     def test_submit_container_output_name(self, stubs):
 572         output_name = "test_output_name"
 573
 574         capture_stdout = cStringIO.StringIO()
 575         try:
 576             exited = arvados_cwl.main(
 577                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 578                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 579                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 580             self.assertEqual(exited, 0)
 581         except:
 582             logging.exception("")
 583
 584         expect_container = copy.deepcopy(stubs.expect_container_spec)
 585         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 586                                                   "--output-name="+output_name, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 587                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 588         expect_container["output_name"] = output_name
 589
 590         stubs.api.container_requests().create.assert_called_with(
 591             body=JsonDiffMatcher(expect_container))
 592         self.assertEqual(capture_stdout.getvalue(),
 593                          stubs.expect_container_request_uuid + '\n')
 594
 595
 596     @stubs
 597     def test_submit_container_output_ttl(self, stubs):
 598         capture_stdout = cStringIO.StringIO()
 599         try:
 600             exited = arvados_cwl.main(
 601                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 602                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 603                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 604             self.assertEqual(exited, 0)
 605         except:
 606             logging.exception("")
 607
 608         expect_container = copy.deepcopy(stubs.expect_container_spec)
 609         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 610                                        '--enable-reuse', '--on-error=continue',
 611                                        "--intermediate-output-ttl=3600", '--eval-timeout=20',
 612                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 613
 614         stubs.api.container_requests().create.assert_called_with(
 615             body=JsonDiffMatcher(expect_container))
 616         self.assertEqual(capture_stdout.getvalue(),
 617                          stubs.expect_container_request_uuid + '\n')
 618
 619     @stubs
 620     def test_submit_container_trash_intermediate(self, stubs):
 621         capture_stdout = cStringIO.StringIO()
 622         try:
 623             exited = arvados_cwl.main(
 624                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 625                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 626                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 627             self.assertEqual(exited, 0)
 628         except:
 629             logging.exception("")
 630
 631         expect_container = copy.deepcopy(stubs.expect_container_spec)
 632         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 633                                        '--enable-reuse', '--on-error=continue',
 634                                        "--trash-intermediate", '--eval-timeout=20',
 635                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 636
 637         stubs.api.container_requests().create.assert_called_with(
 638             body=JsonDiffMatcher(expect_container))
 639         self.assertEqual(capture_stdout.getvalue(),
 640                          stubs.expect_container_request_uuid + '\n')
 641
 642     @stubs
 643     def test_submit_container_output_tags(self, stubs):
 644         output_tags = "tag0,tag1,tag2"
 645
 646         capture_stdout = cStringIO.StringIO()
 647         try:
 648             exited = arvados_cwl.main(
 649                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 650                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 651                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 652             self.assertEqual(exited, 0)
 653         except:
 654             logging.exception("")
 655
 656         expect_container = copy.deepcopy(stubs.expect_container_spec)
 657         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 658                                                   "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 659                                                   '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 660
 661         stubs.api.container_requests().create.assert_called_with(
 662             body=JsonDiffMatcher(expect_container))
 663         self.assertEqual(capture_stdout.getvalue(),
 664                          stubs.expect_container_request_uuid + '\n')
 665
 666     @stubs
 667     def test_submit_container_runner_ram(self, stubs):
 668         capture_stdout = cStringIO.StringIO()
 669         try:
 670             exited = arvados_cwl.main(
 671                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 672                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 673                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 674             self.assertEqual(exited, 0)
 675         except:
 676             logging.exception("")
 677
 678         expect_container = copy.deepcopy(stubs.expect_container_spec)
 679         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 680
 681         stubs.api.container_requests().create.assert_called_with(
 682             body=JsonDiffMatcher(expect_container))
 683         self.assertEqual(capture_stdout.getvalue(),
 684                          stubs.expect_container_request_uuid + '\n')
 685
 686     @mock.patch("arvados.collection.CollectionReader")
 687     @mock.patch("time.sleep")
 688     @stubs
 689     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 690         capture_stdout = cStringIO.StringIO()
 691         exited = arvados_cwl.main(
 692             ["--submit", "--no-wait", "--api=containers", "--debug",
 693              "tests/wf/submit_keepref_wf.cwl"],
 694             capture_stdout, sys.stderr, api_client=stubs.api)
 695         self.assertEqual(exited, 0)
 696
 697
 698     @mock.patch("arvados.collection.CollectionReader")
 699     @mock.patch("time.sleep")
 700     @stubs
 701     def test_submit_keepref(self, stubs, tm, reader):
 702         capture_stdout = cStringIO.StringIO()
 703
 704         with open("tests/wf/expect_arvworkflow.cwl") as f:
 705             reader().open().__enter__().read.return_value = f.read()
 706
 707         exited = arvados_cwl.main(
 708             ["--submit", "--no-wait", "--api=containers", "--debug",
 709              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 710             capture_stdout, sys.stderr, api_client=stubs.api)
 711         self.assertEqual(exited, 0)
 712
 713         expect_container = {
 714             'priority': 500,
 715             'mounts': {
 716                 '/var/spool/cwl': {
 717                     'writable': True,
 718                     'kind': 'collection'
 719                 },
 720                 'stdout': {
 721                     'path': '/var/spool/cwl/cwl.output.json',
 722                     'kind': 'file'
 723                 },
 724                 '/var/lib/cwl/workflow': {
 725                     'portable_data_hash': '99999999999999999999999999999994+99',
 726                     'kind': 'collection'
 727                 },
 728                 '/var/lib/cwl/cwl.input.json': {
 729                     'content': {
 730                         'x': 'XxX'
 731                     },
 732                     'kind': 'json'
 733                 }
 734             }, 'state': 'Committed',
 735             'owner_uuid': None,
 736             'output_path': '/var/spool/cwl',
 737             'name': 'expect_arvworkflow.cwl#main',
 738             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 739             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 740                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 741                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 742             'cwd': '/var/spool/cwl',
 743             'runtime_constraints': {
 744                 'API': True,
 745                 'vcpus': 1,
 746                 'ram': 1073741824
 747             },
 748             'use_existing': True,
 749             'properties': {},
 750             'secret_mounts': {}
 751         }
 752
 753         stubs.api.container_requests().create.assert_called_with(
 754             body=JsonDiffMatcher(expect_container))
 755         self.assertEqual(capture_stdout.getvalue(),
 756                          stubs.expect_container_request_uuid + '\n')
 757
 758
 759     @mock.patch("arvados.collection.CollectionReader")
 760     @mock.patch("time.sleep")
 761     @stubs
 762     def test_submit_jobs_keepref(self, stubs, tm, reader):
 763         capture_stdout = cStringIO.StringIO()
 764
 765         with open("tests/wf/expect_arvworkflow.cwl") as f:
 766             reader().open().__enter__().read.return_value = f.read()
 767
 768         exited = arvados_cwl.main(
 769             ["--submit", "--no-wait", "--api=jobs", "--debug",
 770              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 771             capture_stdout, sys.stderr, api_client=stubs.api)
 772         self.assertEqual(exited, 0)
 773
 774         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 775         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 776         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 777         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 778         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 779         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 780         stubs.api.pipeline_instances().create.assert_called_with(
 781             body=JsonDiffMatcher(expect_pipeline))
 782
 783     @mock.patch("time.sleep")
 784     @stubs
 785     def test_submit_arvworkflow(self, stubs, tm):
 786         capture_stdout = cStringIO.StringIO()
 787
 788         with open("tests/wf/expect_arvworkflow.cwl") as f:
 789             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 790
 791         exited = arvados_cwl.main(
 792             ["--submit", "--no-wait", "--api=containers", "--debug",
 793              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 794             capture_stdout, sys.stderr, api_client=stubs.api)
 795         self.assertEqual(exited, 0)
 796
 797         expect_container = {
 798             'priority': 500,
 799             'mounts': {
 800                 '/var/spool/cwl': {
 801                     'writable': True,
 802                     'kind': 'collection'
 803                 },
 804                 'stdout': {
 805                     'path': '/var/spool/cwl/cwl.output.json',
 806                     'kind': 'file'
 807                 },
 808                 '/var/lib/cwl/workflow.json': {
 809                     'kind': 'json',
 810                     'content': {
 811                         'cwlVersion': 'v1.0',
 812                         '$graph': [
 813                             {
 814                                 'id': '#main',
 815                                 'inputs': [
 816                                     {'type': 'string', 'id': '#main/x'}
 817                                 ],
 818                                 'steps': [
 819                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 820                                      'run': '#submit_tool.cwl',
 821                                      'id': '#main/step1',
 822                                      'out': []}
 823                                 ],
 824                                 'class': 'Workflow',
 825                                 'outputs': []
 826                             },
 827                             {
 828                                 'inputs': [
 829                                     {
 830                                         'inputBinding': {'position': 1},
 831                                         'type': 'string',
 832                                         'id': '#submit_tool.cwl/x'}
 833                                 ],
 834                                 'requirements': [
 835                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 836                                 ],
 837                                 'id': '#submit_tool.cwl',
 838                                 'outputs': [],
 839                                 'baseCommand': 'cat',
 840                                 'class': 'CommandLineTool'
 841                             }
 842                         ]
 843                     }
 844                 },
 845                 '/var/lib/cwl/cwl.input.json': {
 846                     'content': {
 847                         'x': 'XxX'
 848                     },
 849                     'kind': 'json'
 850                 }
 851             }, 'state': 'Committed',
 852             'owner_uuid': None,
 853             'output_path': '/var/spool/cwl',
 854             'name': 'a test workflow',
 855             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 856             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 857                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 858                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 859             'cwd': '/var/spool/cwl',
 860             'runtime_constraints': {
 861                 'API': True,
 862                 'vcpus': 1,
 863                 'ram': 1073741824
 864             },
 865             'use_existing': True,
 866             'properties': {
 867                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 868             },
 869             'secret_mounts': {}
 870         }
 871
 872         stubs.api.container_requests().create.assert_called_with(
 873             body=JsonDiffMatcher(expect_container))
 874         self.assertEqual(capture_stdout.getvalue(),
 875                          stubs.expect_container_request_uuid + '\n')
 876
 877
 878     @stubs
 879     def test_submit_container_name(self, stubs):
 880         capture_stdout = cStringIO.StringIO()
 881         try:
 882             exited = arvados_cwl.main(
 883                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 884                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 885                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 886             self.assertEqual(exited, 0)
 887         except:
 888             logging.exception("")
 889
 890         expect_container = copy.deepcopy(stubs.expect_container_spec)
 891         expect_container["name"] = "hello container 123"
 892
 893         stubs.api.container_requests().create.assert_called_with(
 894             body=JsonDiffMatcher(expect_container))
 895         self.assertEqual(capture_stdout.getvalue(),
 896                          stubs.expect_container_request_uuid + '\n')
 897
 898
 899     @stubs
 900     def test_submit_container_project(self, stubs):
 901         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 902         capture_stdout = cStringIO.StringIO()
 903         try:
 904             exited = arvados_cwl.main(
 905                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 906                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 907                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 908             self.assertEqual(exited, 0)
 909         except:
 910             logging.exception("")
 911
 912         expect_container = copy.deepcopy(stubs.expect_container_spec)
 913         expect_container["owner_uuid"] = project_uuid
 914         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 915                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid, '--eval-timeout=20',
 916                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 917
 918         stubs.api.container_requests().create.assert_called_with(
 919             body=JsonDiffMatcher(expect_container))
 920         self.assertEqual(capture_stdout.getvalue(),
 921                          stubs.expect_container_request_uuid + '\n')
 922
 923     @stubs
 924     def test_submit_container_eval_timeout(self, stubs):
 925         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 926         capture_stdout = cStringIO.StringIO()
 927         try:
 928             exited = arvados_cwl.main(
 929                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 930                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 931                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 932             self.assertEqual(exited, 0)
 933         except:
 934             logging.exception("")
 935
 936         expect_container = copy.deepcopy(stubs.expect_container_spec)
 937         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 938                                        '--enable-reuse', '--on-error=continue', '--eval-timeout=60.0',
 939                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 940
 941         stubs.api.container_requests().create.assert_called_with(
 942             body=JsonDiffMatcher(expect_container))
 943         self.assertEqual(capture_stdout.getvalue(),
 944                          stubs.expect_container_request_uuid + '\n')
 945
 946
 947     @stubs
 948     def test_submit_job_runner_image(self, stubs):
 949         capture_stdout = cStringIO.StringIO()
 950         try:
 951             exited = arvados_cwl.main(
 952                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 953                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 954                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 955             self.assertEqual(exited, 0)
 956         except:
 957             logging.exception("")
 958
 959         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 960
 961         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 962         stubs.api.pipeline_instances().create.assert_called_with(
 963             body=JsonDiffMatcher(expect_pipeline))
 964         self.assertEqual(capture_stdout.getvalue(),
 965                          stubs.expect_pipeline_uuid + '\n')
 966
 967     @stubs
 968     def test_submit_container_runner_image(self, stubs):
 969         capture_stdout = cStringIO.StringIO()
 970         try:
 971             exited = arvados_cwl.main(
 972                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 973                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 974                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 975             self.assertEqual(exited, 0)
 976         except:
 977             logging.exception("")
 978
 979         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 980
 981         expect_container = copy.deepcopy(stubs.expect_container_spec)
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986
 987     @stubs
 988     def test_submit_priority(self, stubs):
 989         capture_stdout = cStringIO.StringIO()
 990         try:
 991             exited = arvados_cwl.main(
 992                 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 993                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 995             self.assertEqual(exited, 0)
 996         except:
 997             logging.exception("")
 998
 999         stubs.expect_container_spec["priority"] = 669
1000
1001         expect_container = copy.deepcopy(stubs.expect_container_spec)
1002         stubs.api.container_requests().create.assert_called_with(
1003             body=JsonDiffMatcher(expect_container))
1004         self.assertEqual(capture_stdout.getvalue(),
1005                          stubs.expect_container_request_uuid + '\n')
1006
1007
1008     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1009     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1010     @mock.patch("arvados.api")
1011     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1012         arvrunner = mock.MagicMock()
1013         arvrunner.project_uuid = ""
1014         api.return_value = mock.MagicMock()
1015         arvrunner.api = api.return_value
1016         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1017                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1018                                                                         "link_class": "docker_image_repo+tag",
1019                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1020                                                                         "owner_uuid": "",
1021                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1022                                                             {"items": [{"created_at": "",
1023                                                                         "head_uuid": "",
1024                                                                         "link_class": "docker_image_hash",
1025                                                                         "name": "123456",
1026                                                                         "owner_uuid": "",
1027                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1028         )
1029         find_one_image_hash.return_value = "123456"
1030
1031         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1032                                                                               "owner_uuid": "",
1033                                                                               "manifest_text": "",
1034                                                                               "properties": ""
1035                                                                           }], "items_available": 1, "offset": 0},)
1036         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1037         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1038                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1039
1040     @stubs
1041     def test_submit_secrets(self, stubs):
1042         capture_stdout = cStringIO.StringIO()
1043         try:
1044             exited = arvados_cwl.main(
1045                 ["--submit", "--no-wait", "--api=containers", "--debug",
1046                  "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1047                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1048             self.assertEqual(exited, 0)
1049         except:
1050             logging.exception("")
1051
1052
1053         expect_container = {
1054             "command": [
1055                 "arvados-cwl-runner",
1056                 "--local",
1057                 "--api=containers",
1058                 "--no-log-timestamps",
1059                 "--enable-reuse",
1060                 "--on-error=continue",
1061                 "--eval-timeout=20",
1062                 "/var/lib/cwl/workflow.json#main",
1063                 "/var/lib/cwl/cwl.input.json"
1064             ],
1065             "container_image": "arvados/jobs:"+arvados_cwl.__version__,
1066             "cwd": "/var/spool/cwl",
1067             "mounts": {
1068                 "/var/lib/cwl/cwl.input.json": {
1069                     "content": {
1070                         "pw": {
1071                             "$include": "/secrets/s0"
1072                         }
1073                     },
1074                     "kind": "json"
1075                 },
1076                 "/var/lib/cwl/workflow.json": {
1077                     "content": {
1078                         "$graph": [
1079                             {
1080                                 "$namespaces": {
1081                                     "cwltool": "http://commonwl.org/cwltool#"
1082                                 },
1083                                 "arguments": [
1084                                     "md5sum",
1085                                     "example.conf"
1086                                 ],
1087                                 "class": "CommandLineTool",
1088                                 "hints": [
1089                                     {
1090                                         "class": "http://commonwl.org/cwltool#Secrets",
1091                                         "secrets": [
1092                                             "#secret_job.cwl/pw"
1093                                         ]
1094                                     }
1095                                 ],
1096                                 "id": "#secret_job.cwl",
1097                                 "inputs": [
1098                                     {
1099                                         "id": "#secret_job.cwl/pw",
1100                                         "type": "string"
1101                                     }
1102                                 ],
1103                                 "outputs": [
1104                                     {
1105                                         "id": "#secret_job.cwl/out",
1106                                         "type": "stdout"
1107                                     }
1108                                 ],
1109                                 "stdout": "hashed_example.txt",
1110                                 "requirements": [
1111                                     {
1112                                         "class": "InitialWorkDirRequirement",
1113                                         "listing": [
1114                                             {
1115                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1116                                                 "entryname": "example.conf"
1117                                             }
1118                                         ]
1119                                     }
1120                                 ]
1121                             },
1122                             {
1123                                 "class": "Workflow",
1124                                 "hints": [
1125                                     {
1126                                         "class": "DockerRequirement",
1127                                         "dockerPull": "debian:8"
1128                                     },
1129                                     {
1130                                         "class": "http://commonwl.org/cwltool#Secrets",
1131                                         "secrets": [
1132                                             "#main/pw"
1133                                         ]
1134                                     }
1135                                 ],
1136                                 "id": "#main",
1137                                 "inputs": [
1138                                     {
1139                                         "id": "#main/pw",
1140                                         "type": "string"
1141                                     }
1142                                 ],
1143                                 "outputs": [
1144                                     {
1145                                         "id": "#main/out",
1146                                         "outputSource": "#main/step1/out",
1147                                         "type": "File"
1148                                     }
1149                                 ],
1150                                 "steps": [
1151                                     {
1152                                         "id": "#main/step1",
1153                                         "in": [
1154                                             {
1155                                                 "id": "#main/step1/pw",
1156                                                 "source": "#main/pw"
1157                                             }
1158                                         ],
1159                                         "out": [
1160                                             "#main/step1/out"
1161                                         ],
1162                                         "run": "#secret_job.cwl"
1163                                     }
1164                                 ]
1165                             }
1166                         ],
1167                         "cwlVersion": "v1.0"
1168                     },
1169                     "kind": "json"
1170                 },
1171                 "/var/spool/cwl": {
1172                     "kind": "collection",
1173                     "writable": True
1174                 },
1175                 "stdout": {
1176                     "kind": "file",
1177                     "path": "/var/spool/cwl/cwl.output.json"
1178                 }
1179             },
1180             "name": "secret_wf.cwl",
1181             "output_path": "/var/spool/cwl",
1182             "owner_uuid": None,
1183             "priority": 500,
1184             "properties": {},
1185             "runtime_constraints": {
1186                 "API": True,
1187                 "ram": 1073741824,
1188                 "vcpus": 1
1189             },
1190             "secret_mounts": {
1191                 "/secrets/s0": {
1192                     "content": "blorp",
1193                     "kind": "text"
1194                 }
1195             },
1196             "state": "Committed",
1197             "use_existing": True
1198         }
1199
1200         stubs.api.container_requests().create.assert_called_with(
1201             body=JsonDiffMatcher(expect_container))
1202         self.assertEqual(capture_stdout.getvalue(),
1203                          stubs.expect_container_request_uuid + '\n')
1204
1205
1206 class TestCreateTemplate(unittest.TestCase):
1207     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1208
1209     def _adjust_script_params(self, expect_component):
1210         expect_component['script_parameters']['x'] = {
1211             'dataclass': 'File',
1212             'required': True,
1213             'type': 'File',
1214             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1215         }
1216         expect_component['script_parameters']['y'] = {
1217             'dataclass': 'Collection',
1218             'required': True,
1219             'type': 'Directory',
1220             'value': '99999999999999999999999999999998+99',
1221         }
1222         expect_component['script_parameters']['z'] = {
1223             'dataclass': 'Collection',
1224             'required': True,
1225             'type': 'Directory',
1226         }
1227
1228     @stubs
1229     def test_create(self, stubs):
1230         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1231
1232         capture_stdout = cStringIO.StringIO()
1233
1234         exited = arvados_cwl.main(
1235             ["--create-workflow", "--debug",
1236              "--api=jobs",
1237              "--project-uuid", project_uuid,
1238              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1239             capture_stdout, sys.stderr, api_client=stubs.api)
1240         self.assertEqual(exited, 0)
1241
1242         stubs.api.pipeline_instances().create.refute_called()
1243         stubs.api.jobs().create.refute_called()
1244
1245         expect_component = copy.deepcopy(stubs.expect_job_spec)
1246         self._adjust_script_params(expect_component)
1247         expect_template = {
1248             "components": {
1249                 "submit_wf.cwl": expect_component,
1250             },
1251             "name": "submit_wf.cwl",
1252             "owner_uuid": project_uuid,
1253         }
1254         stubs.api.pipeline_templates().create.assert_called_with(
1255             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1256
1257         self.assertEqual(capture_stdout.getvalue(),
1258                          stubs.expect_pipeline_template_uuid + '\n')
1259
1260
1261     @stubs
1262     def test_create_name(self, stubs):
1263         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1264
1265         capture_stdout = cStringIO.StringIO()
1266
1267         exited = arvados_cwl.main(
1268             ["--create-workflow", "--debug",
1269              "--project-uuid", project_uuid,
1270              "--api=jobs",
1271              "--name", "testing 123",
1272              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1273             capture_stdout, sys.stderr, api_client=stubs.api)
1274         self.assertEqual(exited, 0)
1275
1276         stubs.api.pipeline_instances().create.refute_called()
1277         stubs.api.jobs().create.refute_called()
1278
1279         expect_component = copy.deepcopy(stubs.expect_job_spec)
1280         self._adjust_script_params(expect_component)
1281         expect_template = {
1282             "components": {
1283                 "testing 123": expect_component,
1284             },
1285             "name": "testing 123",
1286             "owner_uuid": project_uuid,
1287         }
1288         stubs.api.pipeline_templates().create.assert_called_with(
1289             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1290
1291         self.assertEqual(capture_stdout.getvalue(),
1292                          stubs.expect_pipeline_template_uuid + '\n')
1293
1294
1295     @stubs
1296     def test_update_name(self, stubs):
1297         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1298
1299         capture_stdout = cStringIO.StringIO()
1300
1301         exited = arvados_cwl.main(
1302             ["--update-workflow", self.existing_template_uuid,
1303              "--debug",
1304              "--project-uuid", project_uuid,
1305              "--api=jobs",
1306              "--name", "testing 123",
1307              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1308             capture_stdout, sys.stderr, api_client=stubs.api)
1309         self.assertEqual(exited, 0)
1310
1311         stubs.api.pipeline_instances().create.refute_called()
1312         stubs.api.jobs().create.refute_called()
1313
1314         expect_component = copy.deepcopy(stubs.expect_job_spec)
1315         self._adjust_script_params(expect_component)
1316         expect_template = {
1317             "components": {
1318                 "testing 123": expect_component,
1319             },
1320             "name": "testing 123",
1321             "owner_uuid": project_uuid,
1322         }
1323         stubs.api.pipeline_templates().create.refute_called()
1324         stubs.api.pipeline_templates().update.assert_called_with(
1325             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1326
1327         self.assertEqual(capture_stdout.getvalue(),
1328                          self.existing_template_uuid + '\n')
1329
1330
1331 class TestCreateWorkflow(unittest.TestCase):
1332     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1333     expect_workflow = StripYAMLComments(
1334         open("tests/wf/expect_packed.cwl").read())
1335
1336     @stubs
1337     def test_create(self, stubs):
1338         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1339
1340         capture_stdout = cStringIO.StringIO()
1341
1342         exited = arvados_cwl.main(
1343             ["--create-workflow", "--debug",
1344              "--api=containers",
1345              "--project-uuid", project_uuid,
1346              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1347             capture_stdout, sys.stderr, api_client=stubs.api)
1348         self.assertEqual(exited, 0)
1349
1350         stubs.api.pipeline_templates().create.refute_called()
1351         stubs.api.container_requests().create.refute_called()
1352
1353         body = {
1354             "workflow": {
1355                 "owner_uuid": project_uuid,
1356                 "name": "submit_wf.cwl",
1357                 "description": "",
1358                 "definition": self.expect_workflow,
1359             }
1360         }
1361         stubs.api.workflows().create.assert_called_with(
1362             body=JsonDiffMatcher(body))
1363
1364         self.assertEqual(capture_stdout.getvalue(),
1365                          stubs.expect_workflow_uuid + '\n')
1366
1367
1368     @stubs
1369     def test_create_name(self, stubs):
1370         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1371
1372         capture_stdout = cStringIO.StringIO()
1373
1374         exited = arvados_cwl.main(
1375             ["--create-workflow", "--debug",
1376              "--api=containers",
1377              "--project-uuid", project_uuid,
1378              "--name", "testing 123",
1379              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1380             capture_stdout, sys.stderr, api_client=stubs.api)
1381         self.assertEqual(exited, 0)
1382
1383         stubs.api.pipeline_templates().create.refute_called()
1384         stubs.api.container_requests().create.refute_called()
1385
1386         body = {
1387             "workflow": {
1388                 "owner_uuid": project_uuid,
1389                 "name": "testing 123",
1390                 "description": "",
1391                 "definition": self.expect_workflow,
1392             }
1393         }
1394         stubs.api.workflows().create.assert_called_with(
1395             body=JsonDiffMatcher(body))
1396
1397         self.assertEqual(capture_stdout.getvalue(),
1398                          stubs.expect_workflow_uuid + '\n')
1399
1400     @stubs
1401     def test_incompatible_api(self, stubs):
1402         capture_stderr = cStringIO.StringIO()
1403         logging.getLogger('arvados.cwl-runner').addHandler(
1404             logging.StreamHandler(capture_stderr))
1405
1406         exited = arvados_cwl.main(
1407             ["--update-workflow", self.existing_workflow_uuid,
1408              "--api=jobs",
1409              "--debug",
1410              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1411             sys.stderr, sys.stderr, api_client=stubs.api)
1412         self.assertEqual(exited, 1)
1413         self.assertRegexpMatches(
1414             capture_stderr.getvalue(),
1415             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1416
1417     @stubs
1418     def test_update(self, stubs):
1419         capture_stdout = cStringIO.StringIO()
1420
1421         exited = arvados_cwl.main(
1422             ["--update-workflow", self.existing_workflow_uuid,
1423              "--debug",
1424              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1425             capture_stdout, sys.stderr, api_client=stubs.api)
1426         self.assertEqual(exited, 0)
1427
1428         body = {
1429             "workflow": {
1430                 "name": "submit_wf.cwl",
1431                 "description": "",
1432                 "definition": self.expect_workflow,
1433             }
1434         }
1435         stubs.api.workflows().update.assert_called_with(
1436             uuid=self.existing_workflow_uuid,
1437             body=JsonDiffMatcher(body))
1438         self.assertEqual(capture_stdout.getvalue(),
1439                          self.existing_workflow_uuid + '\n')
1440
1441
1442     @stubs
1443     def test_update_name(self, stubs):
1444         capture_stdout = cStringIO.StringIO()
1445
1446         exited = arvados_cwl.main(
1447             ["--update-workflow", self.existing_workflow_uuid,
1448              "--debug", "--name", "testing 123",
1449              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1450             capture_stdout, sys.stderr, api_client=stubs.api)
1451         self.assertEqual(exited, 0)
1452
1453         body = {
1454             "workflow": {
1455                 "name": "testing 123",
1456                 "description": "",
1457                 "definition": self.expect_workflow,
1458             }
1459         }
1460         stubs.api.workflows().update.assert_called_with(
1461             uuid=self.existing_workflow_uuid,
1462             body=JsonDiffMatcher(body))
1463         self.assertEqual(capture_stdout.getvalue(),
1464                          self.existing_workflow_uuid + '\n')
1465
1466
1467     @stubs
1468     def test_create_collection_per_tool(self, stubs):
1469         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1470
1471         capture_stdout = cStringIO.StringIO()
1472
1473         exited = arvados_cwl.main(
1474             ["--create-workflow", "--debug",
1475              "--api=containers",
1476              "--project-uuid", project_uuid,
1477              "tests/collection_per_tool/collection_per_tool.cwl"],
1478             capture_stdout, sys.stderr, api_client=stubs.api)
1479         self.assertEqual(exited, 0)
1480
1481         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1482         expect_workflow = StripYAMLComments(open(toolfile).read())
1483
1484         body = {
1485             "workflow": {
1486                 "owner_uuid": project_uuid,
1487                 "name": "collection_per_tool.cwl",
1488                 "description": "",
1489                 "definition": expect_workflow,
1490             }
1491         }
1492         stubs.api.workflows().create.assert_called_with(
1493             body=JsonDiffMatcher(body))
1494
1495         self.assertEqual(capture_stdout.getvalue(),
1496                          stubs.expect_workflow_uuid + '\n')
1497
1498 class TestTemplateInputs(unittest.TestCase):
1499     expect_template = {
1500         "components": {
1501             "inputs_test.cwl": {
1502                 'runtime_constraints': {
1503                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1504                     'min_ram_mb_per_node': 1024
1505                 },
1506                 'script_parameters': {
1507                     'cwl:tool':
1508                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1509                     'optionalFloatInput': None,
1510                     'fileInput': {
1511                         'type': 'File',
1512                         'dataclass': 'File',
1513                         'required': True,
1514                         'title': "It's a file; we expect to find some characters in it.",
1515                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1516                     },
1517                     'floatInput': {
1518                         'type': 'float',
1519                         'dataclass': 'number',
1520                         'required': True,
1521                         'title': 'Floats like a duck',
1522                         'default': 0.1,
1523                         'value': 0.1,
1524                     },
1525                     'optionalFloatInput': {
1526                         'type': ['null', 'float'],
1527                         'dataclass': 'number',
1528                         'required': False,
1529                     },
1530                     'boolInput': {
1531                         'type': 'boolean',
1532                         'dataclass': 'boolean',
1533                         'required': True,
1534                         'title': 'True or false?',
1535                     },
1536                 },
1537                 'repository': 'arvados',
1538                 'script_version': 'master',
1539                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1540                 'script': 'cwl-runner',
1541             },
1542         },
1543         "name": "inputs_test.cwl",
1544     }
1545
1546     @stubs
1547     def test_inputs_empty(self, stubs):
1548         exited = arvados_cwl.main(
1549             ["--create-template",
1550              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1551             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1552         self.assertEqual(exited, 0)
1553
1554         stubs.api.pipeline_templates().create.assert_called_with(
1555             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1556
1557     @stubs
1558     def test_inputs(self, stubs):
1559         exited = arvados_cwl.main(
1560             ["--create-template",
1561              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1562             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1563         self.assertEqual(exited, 0)
1564
1565         expect_template = copy.deepcopy(self.expect_template)
1566         params = expect_template[
1567             "components"]["inputs_test.cwl"]["script_parameters"]
1568         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1569         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1570         params["floatInput"]["value"] = 1.234
1571         params["boolInput"]["value"] = True
1572
1573         stubs.api.pipeline_templates().create.assert_called_with(
1574             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)