sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import json
  10 import logging
  11 import mock
  12 import sys
  13 import unittest
  14
  15 import arvados
  16 import arvados.collection
  17 import arvados_cwl
  18 import arvados_cwl.runner
  19 import arvados.keep
  20
  21 from .matcher import JsonDiffMatcher, StripYAMLComments
  22 from .mock_discovery import get_rootDesc
  23
  24 import ruamel.yaml as yaml
  25
  26 _rootDesc = None
  27
  28 def stubs(func):
  29     @functools.wraps(func)
  30     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  31     @mock.patch("arvados.collection.KeepClient")
  32     @mock.patch("arvados.keep.KeepClient")
  33     @mock.patch("arvados.events.subscribe")
  34     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  35         class Stubs:
  36             pass
  37         stubs = Stubs()
  38         stubs.events = events
  39         stubs.keepdocker = keepdocker
  40
  41         def putstub(p, **kwargs):
  42             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  43         keep_client1().put.side_effect = putstub
  44         keep_client1.put.side_effect = putstub
  45         keep_client2().put.side_effect = putstub
  46         keep_client2.put.side_effect = putstub
  47
  48         stubs.keep_client = keep_client2
  49         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  50         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  51
  52         stubs.api = mock.MagicMock()
  53         stubs.api._rootDesc = get_rootDesc()
  54
  55         stubs.api.users().current().execute.return_value = {
  56             "uuid": stubs.fake_user_uuid,
  57         }
  58         stubs.api.collections().list().execute.return_value = {"items": []}
  59
  60         class CollectionExecute(object):
  61             def __init__(self, exe):
  62                 self.exe = exe
  63             def execute(self, num_retries=None):
  64                 return self.exe
  65
  66         def collection_createstub(created_collections, body, ensure_unique_name=None):
  67             mt = body["manifest_text"]
  68             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
  69             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
  70             created_collections[uuid] = {
  71                 "uuid": uuid,
  72                 "portable_data_hash": pdh,
  73                 "manifest_text": mt
  74             }
  75             return CollectionExecute(created_collections[uuid])
  76
  77         def collection_getstub(created_collections, uuid):
  78             for v in created_collections.itervalues():
  79                 if uuid in (v["uuid"], v["portable_data_hash"]):
  80                     return CollectionExecute(v)
  81
  82         created_collections = {
  83             "99999999999999999999999999999998+99": {
  84                 "uuid": "",
  85                 "portable_data_hash": "99999999999999999999999999999998+99",
  86                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
  87             },
  88             "99999999999999999999999999999994+99": {
  89                 "uuid": "",
  90                 "portable_data_hash": "99999999999999999999999999999994+99",
  91                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
  92             }
  93         }
  94         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
  95         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
  96
  97         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  98         stubs.api.jobs().create().execute.return_value = {
  99             "uuid": stubs.expect_job_uuid,
 100             "state": "Queued",
 101         }
 102
 103         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 104         stubs.api.container_requests().create().execute.return_value = {
 105             "uuid": stubs.expect_container_request_uuid,
 106             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 107             "state": "Queued"
 108         }
 109
 110         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 111         stubs.api.pipeline_templates().create().execute.return_value = {
 112             "uuid": stubs.expect_pipeline_template_uuid,
 113         }
 114         stubs.expect_job_spec = {
 115             'runtime_constraints': {
 116                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 117                 'min_ram_mb_per_node': 1024
 118             },
 119             'script_parameters': {
 120                 'x': {
 121                     'basename': 'blorp.txt',
 122                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 123                     'class': 'File'
 124                 },
 125                 'y': {
 126                     'basename': '99999999999999999999999999999998+99',
 127                     'location': 'keep:99999999999999999999999999999998+99',
 128                     'class': 'Directory'
 129                 },
 130                 'z': {
 131                     'basename': 'anonymous',
 132                     "listing": [{
 133                         "basename": "renamed.txt",
 134                         "class": "File",
 135                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 136                     }],
 137                     'class': 'Directory'
 138                 },
 139                 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
 140             },
 141             'repository': 'arvados',
 142             'script_version': 'master',
 143             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 144             'script': 'cwl-runner'
 145         }
 146         stubs.pipeline_component = stubs.expect_job_spec.copy()
 147         stubs.expect_pipeline_instance = {
 148             'name': 'submit_wf.cwl',
 149             'state': 'RunningOnServer',
 150             'owner_uuid': None,
 151             "components": {
 152                 "cwl-runner": {
 153                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 154                     'script_parameters': {
 155                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 156                         'x': {"value": {
 157                             'basename': 'blorp.txt',
 158                             'class': 'File',
 159                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 160                             "size": 16
 161                         }},
 162                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 163                               'listing': [
 164                                   {
 165                                       'basename': 'renamed.txt',
 166                                       'class': 'File', 'location':
 167                                       'keep:99999999999999999999999999999998+99/file1.txt'
 168                                   }
 169                               ]}},
 170                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
 171                         'arv:enable_reuse': True,
 172                         'arv:on_error': 'continue'
 173                     },
 174                     'repository': 'arvados',
 175                     'script_version': 'master',
 176                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 177                     'script': 'cwl-runner',
 178                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 179                 }
 180             }
 181         }
 182         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 183         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 184         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 185         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 186         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 187             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 188             "state": "Queued"
 189         }
 190         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 191         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 192
 193         with open("tests/wf/submit_wf_packed.cwl") as f:
 194             expect_packed_workflow = yaml.round_trip_load(f)
 195
 196         stubs.expect_container_spec = {
 197             'priority': 500,
 198             'mounts': {
 199                 '/var/spool/cwl': {
 200                     'writable': True,
 201                     'kind': 'collection'
 202                 },
 203                 '/var/lib/cwl/workflow.json': {
 204                     'content': expect_packed_workflow,
 205                     'kind': 'json'
 206                 },
 207                 'stdout': {
 208                     'path': '/var/spool/cwl/cwl.output.json',
 209                     'kind': 'file'
 210                 },
 211                 '/var/lib/cwl/cwl.input.json': {
 212                     'kind': 'json',
 213                     'content': {
 214                         'y': {
 215                             'basename': '99999999999999999999999999999998+99',
 216                             'location': 'keep:99999999999999999999999999999998+99',
 217                             'class': 'Directory'},
 218                         'x': {
 219                             'basename': u'blorp.txt',
 220                             'class': 'File',
 221                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 222                             "size": 16
 223                         },
 224                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 225                             {'basename': 'renamed.txt',
 226                              'class': 'File',
 227                              'location': 'keep:99999999999999999999999999999998+99/file1.txt'
 228                             }
 229                         ]}
 230                     },
 231                     'kind': 'json'
 232                 }
 233             },
 234             'secret_mounts': {},
 235             'state': 'Committed',
 236             'owner_uuid': None,
 237             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 238                         '--no-log-timestamps', '--disable-validate',
 239                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 240                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 241             'name': 'submit_wf.cwl',
 242             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 243             'output_path': '/var/spool/cwl',
 244             'cwd': '/var/spool/cwl',
 245             'runtime_constraints': {
 246                 'API': True,
 247                 'vcpus': 1,
 248                 'ram': 1024*1024*1024
 249             },
 250             'use_existing': True,
 251             'properties': {},
 252             'secret_mounts': {}
 253         }
 254
 255         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 256         stubs.api.workflows().create().execute.return_value = {
 257             "uuid": stubs.expect_workflow_uuid,
 258         }
 259         def update_mock(**kwargs):
 260             stubs.updated_uuid = kwargs.get('uuid')
 261             return mock.DEFAULT
 262         stubs.api.workflows().update.side_effect = update_mock
 263         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 264             "uuid": stubs.updated_uuid,
 265         }
 266
 267         return func(self, stubs, *args, **kwargs)
 268     return wrapped
 269
 270
 271 class TestSubmit(unittest.TestCase):
 272     @mock.patch("arvados_cwl.runner.arv_docker_get_image")
 273     @mock.patch("time.sleep")
 274     @stubs
 275     def test_submit(self, stubs, tm, arvdock):
 276         capture_stdout = cStringIO.StringIO()
 277         exited = arvados_cwl.main(
 278             ["--submit", "--no-wait", "--api=jobs", "--debug",
 279              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 280             capture_stdout, sys.stderr, api_client=stubs.api)
 281         self.assertEqual(exited, 0)
 282
 283         stubs.api.collections().create.assert_has_calls([
 284             mock.call(body=JsonDiffMatcher({
 285                 'manifest_text':
 286                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 287                 'replication_desired': None,
 288                 'name': 'submit_tool.cwl dependencies',
 289             }), ensure_unique_name=True),
 290             mock.call(body=JsonDiffMatcher({
 291                 'manifest_text':
 292                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 293                 'replication_desired': None,
 294                 'name': 'submit_wf.cwl input',
 295             }), ensure_unique_name=True),
 296             mock.call(body=JsonDiffMatcher({
 297                 'manifest_text':
 298                 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
 299                 'replication_desired': None,
 300                 'name': 'submit_wf.cwl',
 301             }), ensure_unique_name=True)        ])
 302
 303         arvdock.assert_has_calls([
 304             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 305             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 306         ])
 307
 308         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 309         stubs.api.pipeline_instances().create.assert_called_with(
 310             body=JsonDiffMatcher(expect_pipeline))
 311         self.assertEqual(capture_stdout.getvalue(),
 312                          stubs.expect_pipeline_uuid + '\n')
 313
 314
 315     @mock.patch("time.sleep")
 316     @stubs
 317     def test_submit_no_reuse(self, stubs, tm):
 318         capture_stdout = cStringIO.StringIO()
 319         exited = arvados_cwl.main(
 320             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 321              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 322             capture_stdout, sys.stderr, api_client=stubs.api)
 323         self.assertEqual(exited, 0)
 324
 325         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 326         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 327         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 328
 329         stubs.api.pipeline_instances().create.assert_called_with(
 330             body=JsonDiffMatcher(expect_pipeline))
 331         self.assertEqual(capture_stdout.getvalue(),
 332                          stubs.expect_pipeline_uuid + '\n')
 333
 334     @mock.patch("time.sleep")
 335     @stubs
 336     def test_submit_on_error(self, stubs, tm):
 337         capture_stdout = cStringIO.StringIO()
 338         exited = arvados_cwl.main(
 339             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 340              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 341             capture_stdout, sys.stderr, api_client=stubs.api)
 342         self.assertEqual(exited, 0)
 343
 344         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 345         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 346
 347         stubs.api.pipeline_instances().create.assert_called_with(
 348             body=JsonDiffMatcher(expect_pipeline))
 349         self.assertEqual(capture_stdout.getvalue(),
 350                          stubs.expect_pipeline_uuid + '\n')
 351
 352
 353     @mock.patch("time.sleep")
 354     @stubs
 355     def test_submit_runner_ram(self, stubs, tm):
 356         capture_stdout = cStringIO.StringIO()
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 360             capture_stdout, sys.stderr, api_client=stubs.api)
 361         self.assertEqual(exited, 0)
 362
 363         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 364         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 365
 366         stubs.api.pipeline_instances().create.assert_called_with(
 367             body=JsonDiffMatcher(expect_pipeline))
 368         self.assertEqual(capture_stdout.getvalue(),
 369                          stubs.expect_pipeline_uuid + '\n')
 370
 371
 372     @mock.patch("time.sleep")
 373     @stubs
 374     def test_submit_invalid_runner_ram(self, stubs, tm):
 375         capture_stdout = cStringIO.StringIO()
 376         exited = arvados_cwl.main(
 377             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 378              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 379             capture_stdout, sys.stderr, api_client=stubs.api)
 380         self.assertEqual(exited, 1)
 381
 382     @mock.patch("time.sleep")
 383     @stubs
 384     def test_submit_output_name(self, stubs, tm):
 385         output_name = "test_output_name"
 386
 387         capture_stdout = cStringIO.StringIO()
 388         exited = arvados_cwl.main(
 389             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 390              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 391             capture_stdout, sys.stderr, api_client=stubs.api)
 392         self.assertEqual(exited, 0)
 393
 394         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 395         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 396
 397         stubs.api.pipeline_instances().create.assert_called_with(
 398             body=JsonDiffMatcher(expect_pipeline))
 399         self.assertEqual(capture_stdout.getvalue(),
 400                          stubs.expect_pipeline_uuid + '\n')
 401
 402
 403     @mock.patch("time.sleep")
 404     @stubs
 405     def test_submit_pipeline_name(self, stubs, tm):
 406         capture_stdout = cStringIO.StringIO()
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 409              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             capture_stdout, sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 414         expect_pipeline["name"] = "hello job 123"
 415
 416         stubs.api.pipeline_instances().create.assert_called_with(
 417             body=JsonDiffMatcher(expect_pipeline))
 418         self.assertEqual(capture_stdout.getvalue(),
 419                          stubs.expect_pipeline_uuid + '\n')
 420
 421     @mock.patch("time.sleep")
 422     @stubs
 423     def test_submit_output_tags(self, stubs, tm):
 424         output_tags = "tag0,tag1,tag2"
 425
 426         capture_stdout = cStringIO.StringIO()
 427         exited = arvados_cwl.main(
 428             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 429              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 430             capture_stdout, sys.stderr, api_client=stubs.api)
 431         self.assertEqual(exited, 0)
 432
 433         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 434         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 435
 436         stubs.api.pipeline_instances().create.assert_called_with(
 437             body=JsonDiffMatcher(expect_pipeline))
 438         self.assertEqual(capture_stdout.getvalue(),
 439                          stubs.expect_pipeline_uuid + '\n')
 440
 441     @mock.patch("time.sleep")
 442     @stubs
 443     def test_submit_with_project_uuid(self, stubs, tm):
 444         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 445
 446         exited = arvados_cwl.main(
 447             ["--submit", "--no-wait",
 448              "--project-uuid", project_uuid,
 449              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 450             sys.stdout, sys.stderr, api_client=stubs.api)
 451         self.assertEqual(exited, 0)
 452
 453         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 454         expect_pipeline["owner_uuid"] = project_uuid
 455         stubs.api.pipeline_instances().create.assert_called_with(
 456             body=JsonDiffMatcher(expect_pipeline))
 457
 458     @stubs
 459     def test_submit_container(self, stubs):
 460         capture_stdout = cStringIO.StringIO()
 461         try:
 462             exited = arvados_cwl.main(
 463                 ["--submit", "--no-wait", "--api=containers", "--debug",
 464                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 465                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 466             self.assertEqual(exited, 0)
 467         except:
 468             logging.exception("")
 469
 470         stubs.api.collections().create.assert_has_calls([
 471             mock.call(body=JsonDiffMatcher({
 472                 'manifest_text':
 473                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 474                 'replication_desired': None,
 475                 'name': 'submit_tool.cwl dependencies',
 476             }), ensure_unique_name=True),
 477             mock.call(body=JsonDiffMatcher({
 478                 'manifest_text':
 479                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 480                 'replication_desired': None,
 481                 'name': 'submit_wf.cwl input',
 482             }), ensure_unique_name=True)])
 483
 484         expect_container = copy.deepcopy(stubs.expect_container_spec)
 485         stubs.api.container_requests().create.assert_called_with(
 486             body=JsonDiffMatcher(expect_container))
 487         self.assertEqual(capture_stdout.getvalue(),
 488                          stubs.expect_container_request_uuid + '\n')
 489
 490     @stubs
 491     def test_submit_container_no_reuse(self, stubs):
 492         capture_stdout = cStringIO.StringIO()
 493         try:
 494             exited = arvados_cwl.main(
 495                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 496                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 497                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 498             self.assertEqual(exited, 0)
 499         except:
 500             logging.exception("")
 501
 502         expect_container = copy.deepcopy(stubs.expect_container_spec)
 503         expect_container["command"] = [
 504             'arvados-cwl-runner', '--local', '--api=containers',
 505             '--no-log-timestamps', '--disable-validate',
 506             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 507             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 508         expect_container["use_existing"] = False
 509
 510         stubs.api.container_requests().create.assert_called_with(
 511             body=JsonDiffMatcher(expect_container))
 512         self.assertEqual(capture_stdout.getvalue(),
 513                          stubs.expect_container_request_uuid + '\n')
 514
 515
 516     @stubs
 517     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 518         capture_stdout = cStringIO.StringIO()
 519
 520         exited = arvados_cwl.main(
 521             ["--submit", "--no-wait", "--api=containers", "--debug",
 522              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 523             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 524         self.assertEqual(exited, 0)
 525
 526         expect_container = copy.deepcopy(stubs.expect_container_spec)
 527         expect_container["command"] = [
 528             'arvados-cwl-runner', '--local', '--api=containers',
 529             '--no-log-timestamps', '--disable-validate',
 530             '--disable-reuse', '--on-error=continue', '--eval-timeout=20',
 531             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 532         expect_container["use_existing"] = False
 533         expect_container["name"] = "submit_wf_no_reuse.cwl"
 534         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 535             {
 536                 "class": "http://arvados.org/cwl#ReuseRequirement",
 537                 "enableReuse": False,
 538             },
 539         ]
 540         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 541             "arv": "http://arvados.org/cwl#",
 542             "cwltool": "http://commonwl.org/cwltool#"
 543         }
 544
 545         stubs.api.container_requests().create.assert_called_with(
 546             body=JsonDiffMatcher(expect_container))
 547         self.assertEqual(capture_stdout.getvalue(),
 548                          stubs.expect_container_request_uuid + '\n')
 549
 550
 551     @stubs
 552     def test_submit_container_on_error(self, stubs):
 553         capture_stdout = cStringIO.StringIO()
 554         try:
 555             exited = arvados_cwl.main(
 556                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 557                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 558                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 559             self.assertEqual(exited, 0)
 560         except:
 561             logging.exception("")
 562
 563         expect_container = copy.deepcopy(stubs.expect_container_spec)
 564         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 565                                        '--no-log-timestamps', '--disable-validate',
 566                                        '--enable-reuse', '--on-error=stop', '--eval-timeout=20',
 567                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 568
 569         stubs.api.container_requests().create.assert_called_with(
 570             body=JsonDiffMatcher(expect_container))
 571         self.assertEqual(capture_stdout.getvalue(),
 572                          stubs.expect_container_request_uuid + '\n')
 573
 574     @stubs
 575     def test_submit_container_output_name(self, stubs):
 576         output_name = "test_output_name"
 577
 578         capture_stdout = cStringIO.StringIO()
 579         try:
 580             exited = arvados_cwl.main(
 581                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 582                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 583                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 584             self.assertEqual(exited, 0)
 585         except:
 586             logging.exception("")
 587
 588         expect_container = copy.deepcopy(stubs.expect_container_spec)
 589         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 590                                        '--no-log-timestamps', '--disable-validate',
 591                                        "--output-name="+output_name, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 592                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 593         expect_container["output_name"] = output_name
 594
 595         stubs.api.container_requests().create.assert_called_with(
 596             body=JsonDiffMatcher(expect_container))
 597         self.assertEqual(capture_stdout.getvalue(),
 598                          stubs.expect_container_request_uuid + '\n')
 599
 600
 601     @stubs
 602     def test_submit_container_output_ttl(self, stubs):
 603         capture_stdout = cStringIO.StringIO()
 604         try:
 605             exited = arvados_cwl.main(
 606                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 607                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 608                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 609             self.assertEqual(exited, 0)
 610         except:
 611             logging.exception("")
 612
 613         expect_container = copy.deepcopy(stubs.expect_container_spec)
 614         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 615                                        '--no-log-timestamps', '--disable-validate',
 616                                        '--enable-reuse', '--on-error=continue',
 617                                        "--intermediate-output-ttl=3600", '--eval-timeout=20',
 618                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 619
 620         stubs.api.container_requests().create.assert_called_with(
 621             body=JsonDiffMatcher(expect_container))
 622         self.assertEqual(capture_stdout.getvalue(),
 623                          stubs.expect_container_request_uuid + '\n')
 624
 625     @stubs
 626     def test_submit_container_trash_intermediate(self, stubs):
 627         capture_stdout = cStringIO.StringIO()
 628         try:
 629             exited = arvados_cwl.main(
 630                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 631                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 632                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 633             self.assertEqual(exited, 0)
 634         except:
 635             logging.exception("")
 636
 637         expect_container = copy.deepcopy(stubs.expect_container_spec)
 638         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 639                                        '--no-log-timestamps', '--disable-validate',
 640                                        '--enable-reuse', '--on-error=continue',
 641                                        "--trash-intermediate", '--eval-timeout=20',
 642                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 643
 644         stubs.api.container_requests().create.assert_called_with(
 645             body=JsonDiffMatcher(expect_container))
 646         self.assertEqual(capture_stdout.getvalue(),
 647                          stubs.expect_container_request_uuid + '\n')
 648
 649     @stubs
 650     def test_submit_container_output_tags(self, stubs):
 651         output_tags = "tag0,tag1,tag2"
 652
 653         capture_stdout = cStringIO.StringIO()
 654         try:
 655             exited = arvados_cwl.main(
 656                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 657                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 658                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 659             self.assertEqual(exited, 0)
 660         except:
 661             logging.exception("")
 662
 663         expect_container = copy.deepcopy(stubs.expect_container_spec)
 664         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 665                                        '--no-log-timestamps', '--disable-validate',
 666                                        "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 667                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 668
 669         stubs.api.container_requests().create.assert_called_with(
 670             body=JsonDiffMatcher(expect_container))
 671         self.assertEqual(capture_stdout.getvalue(),
 672                          stubs.expect_container_request_uuid + '\n')
 673
 674     @stubs
 675     def test_submit_container_runner_ram(self, stubs):
 676         capture_stdout = cStringIO.StringIO()
 677         try:
 678             exited = arvados_cwl.main(
 679                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 680                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 681                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 682             self.assertEqual(exited, 0)
 683         except:
 684             logging.exception("")
 685
 686         expect_container = copy.deepcopy(stubs.expect_container_spec)
 687         expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
 688
 689         stubs.api.container_requests().create.assert_called_with(
 690             body=JsonDiffMatcher(expect_container))
 691         self.assertEqual(capture_stdout.getvalue(),
 692                          stubs.expect_container_request_uuid + '\n')
 693
 694     @mock.patch("arvados.collection.CollectionReader")
 695     @mock.patch("time.sleep")
 696     @stubs
 697     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 698         capture_stdout = cStringIO.StringIO()
 699         exited = arvados_cwl.main(
 700             ["--submit", "--no-wait", "--api=containers", "--debug",
 701              "tests/wf/submit_keepref_wf.cwl"],
 702             capture_stdout, sys.stderr, api_client=stubs.api)
 703         self.assertEqual(exited, 0)
 704
 705
 706     @mock.patch("arvados.collection.CollectionReader")
 707     @mock.patch("time.sleep")
 708     @stubs
 709     def test_submit_keepref(self, stubs, tm, reader):
 710         capture_stdout = cStringIO.StringIO()
 711
 712         with open("tests/wf/expect_arvworkflow.cwl") as f:
 713             reader().open().__enter__().read.return_value = f.read()
 714
 715         exited = arvados_cwl.main(
 716             ["--submit", "--no-wait", "--api=containers", "--debug",
 717              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 718             capture_stdout, sys.stderr, api_client=stubs.api)
 719         self.assertEqual(exited, 0)
 720
 721         expect_container = {
 722             'priority': 500,
 723             'mounts': {
 724                 '/var/spool/cwl': {
 725                     'writable': True,
 726                     'kind': 'collection'
 727                 },
 728                 'stdout': {
 729                     'path': '/var/spool/cwl/cwl.output.json',
 730                     'kind': 'file'
 731                 },
 732                 '/var/lib/cwl/workflow': {
 733                     'portable_data_hash': '99999999999999999999999999999994+99',
 734                     'kind': 'collection'
 735                 },
 736                 '/var/lib/cwl/cwl.input.json': {
 737                     'content': {
 738                         'x': 'XxX'
 739                     },
 740                     'kind': 'json'
 741                 }
 742             }, 'state': 'Committed',
 743             'owner_uuid': None,
 744             'output_path': '/var/spool/cwl',
 745             'name': 'expect_arvworkflow.cwl#main',
 746             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 747             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 748                         '--no-log-timestamps', '--disable-validate',
 749                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 750                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 751             'cwd': '/var/spool/cwl',
 752             'runtime_constraints': {
 753                 'API': True,
 754                 'vcpus': 1,
 755                 'ram': 1073741824
 756             },
 757             'use_existing': True,
 758             'properties': {},
 759             'secret_mounts': {}
 760         }
 761
 762         stubs.api.container_requests().create.assert_called_with(
 763             body=JsonDiffMatcher(expect_container))
 764         self.assertEqual(capture_stdout.getvalue(),
 765                          stubs.expect_container_request_uuid + '\n')
 766
 767
 768     @mock.patch("arvados.collection.CollectionReader")
 769     @mock.patch("time.sleep")
 770     @stubs
 771     def test_submit_jobs_keepref(self, stubs, tm, reader):
 772         capture_stdout = cStringIO.StringIO()
 773
 774         with open("tests/wf/expect_arvworkflow.cwl") as f:
 775             reader().open().__enter__().read.return_value = f.read()
 776
 777         exited = arvados_cwl.main(
 778             ["--submit", "--no-wait", "--api=jobs", "--debug",
 779              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 780             capture_stdout, sys.stderr, api_client=stubs.api)
 781         self.assertEqual(exited, 0)
 782
 783         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 784         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 785         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 786         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 787         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 788         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 789         stubs.api.pipeline_instances().create.assert_called_with(
 790             body=JsonDiffMatcher(expect_pipeline))
 791
 792     @mock.patch("time.sleep")
 793     @stubs
 794     def test_submit_arvworkflow(self, stubs, tm):
 795         capture_stdout = cStringIO.StringIO()
 796
 797         with open("tests/wf/expect_arvworkflow.cwl") as f:
 798             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 799
 800         exited = arvados_cwl.main(
 801             ["--submit", "--no-wait", "--api=containers", "--debug",
 802              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 803             capture_stdout, sys.stderr, api_client=stubs.api)
 804         self.assertEqual(exited, 0)
 805
 806         expect_container = {
 807             'priority': 500,
 808             'mounts': {
 809                 '/var/spool/cwl': {
 810                     'writable': True,
 811                     'kind': 'collection'
 812                 },
 813                 'stdout': {
 814                     'path': '/var/spool/cwl/cwl.output.json',
 815                     'kind': 'file'
 816                 },
 817                 '/var/lib/cwl/workflow.json': {
 818                     'kind': 'json',
 819                     'content': {
 820                         'cwlVersion': 'v1.0',
 821                         '$graph': [
 822                             {
 823                                 'id': '#main',
 824                                 'inputs': [
 825                                     {'type': 'string', 'id': '#main/x'}
 826                                 ],
 827                                 'steps': [
 828                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 829                                      'run': '#submit_tool.cwl',
 830                                      'id': '#main/step1',
 831                                      'out': []}
 832                                 ],
 833                                 'class': 'Workflow',
 834                                 'outputs': []
 835                             },
 836                             {
 837                                 'inputs': [
 838                                     {
 839                                         'inputBinding': {'position': 1},
 840                                         'type': 'string',
 841                                         'id': '#submit_tool.cwl/x'}
 842                                 ],
 843                                 'requirements': [
 844                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 845                                 ],
 846                                 'id': '#submit_tool.cwl',
 847                                 'outputs': [],
 848                                 'baseCommand': 'cat',
 849                                 'class': 'CommandLineTool'
 850                             }
 851                         ]
 852                     }
 853                 },
 854                 '/var/lib/cwl/cwl.input.json': {
 855                     'content': {
 856                         'x': 'XxX'
 857                     },
 858                     'kind': 'json'
 859                 }
 860             }, 'state': 'Committed',
 861             'owner_uuid': None,
 862             'output_path': '/var/spool/cwl',
 863             'name': 'a test workflow',
 864             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 865             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 866                         '--no-log-timestamps', '--disable-validate',
 867                         '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
 868                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 869             'cwd': '/var/spool/cwl',
 870             'runtime_constraints': {
 871                 'API': True,
 872                 'vcpus': 1,
 873                 'ram': 1073741824
 874             },
 875             'use_existing': True,
 876             'properties': {
 877                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 878             },
 879             'secret_mounts': {}
 880         }
 881
 882         stubs.api.container_requests().create.assert_called_with(
 883             body=JsonDiffMatcher(expect_container))
 884         self.assertEqual(capture_stdout.getvalue(),
 885                          stubs.expect_container_request_uuid + '\n')
 886
 887
 888     @stubs
 889     def test_submit_container_name(self, stubs):
 890         capture_stdout = cStringIO.StringIO()
 891         try:
 892             exited = arvados_cwl.main(
 893                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 894                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 895                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 896             self.assertEqual(exited, 0)
 897         except:
 898             logging.exception("")
 899
 900         expect_container = copy.deepcopy(stubs.expect_container_spec)
 901         expect_container["name"] = "hello container 123"
 902
 903         stubs.api.container_requests().create.assert_called_with(
 904             body=JsonDiffMatcher(expect_container))
 905         self.assertEqual(capture_stdout.getvalue(),
 906                          stubs.expect_container_request_uuid + '\n')
 907
 908
 909     @stubs
 910     def test_submit_container_project(self, stubs):
 911         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 912         capture_stdout = cStringIO.StringIO()
 913         try:
 914             exited = arvados_cwl.main(
 915                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 916                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 917                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 918             self.assertEqual(exited, 0)
 919         except:
 920             logging.exception("")
 921
 922         expect_container = copy.deepcopy(stubs.expect_container_spec)
 923         expect_container["owner_uuid"] = project_uuid
 924         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 925                                        '--no-log-timestamps', '--disable-validate',
 926                                        '--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid, '--eval-timeout=20',
 927                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 928
 929         stubs.api.container_requests().create.assert_called_with(
 930             body=JsonDiffMatcher(expect_container))
 931         self.assertEqual(capture_stdout.getvalue(),
 932                          stubs.expect_container_request_uuid + '\n')
 933
 934     @stubs
 935     def test_submit_container_eval_timeout(self, stubs):
 936         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 937         capture_stdout = cStringIO.StringIO()
 938         try:
 939             exited = arvados_cwl.main(
 940                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 941                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 942                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 943             self.assertEqual(exited, 0)
 944         except:
 945             logging.exception("")
 946
 947         expect_container = copy.deepcopy(stubs.expect_container_spec)
 948         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 949                                        '--no-log-timestamps', '--disable-validate',
 950                                        '--enable-reuse', '--on-error=continue', '--eval-timeout=60.0',
 951                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 952
 953         stubs.api.container_requests().create.assert_called_with(
 954             body=JsonDiffMatcher(expect_container))
 955         self.assertEqual(capture_stdout.getvalue(),
 956                          stubs.expect_container_request_uuid + '\n')
 957
 958
 959     @stubs
 960     def test_submit_job_runner_image(self, stubs):
 961         capture_stdout = cStringIO.StringIO()
 962         try:
 963             exited = arvados_cwl.main(
 964                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
 965                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 966                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 967             self.assertEqual(exited, 0)
 968         except:
 969             logging.exception("")
 970
 971         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
 972
 973         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 974         stubs.api.pipeline_instances().create.assert_called_with(
 975             body=JsonDiffMatcher(expect_pipeline))
 976         self.assertEqual(capture_stdout.getvalue(),
 977                          stubs.expect_pipeline_uuid + '\n')
 978
 979     @stubs
 980     def test_submit_container_runner_image(self, stubs):
 981         capture_stdout = cStringIO.StringIO()
 982         try:
 983             exited = arvados_cwl.main(
 984                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 985                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 986                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 987             self.assertEqual(exited, 0)
 988         except:
 989             logging.exception("")
 990
 991         stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
 992
 993         expect_container = copy.deepcopy(stubs.expect_container_spec)
 994         stubs.api.container_requests().create.assert_called_with(
 995             body=JsonDiffMatcher(expect_container))
 996         self.assertEqual(capture_stdout.getvalue(),
 997                          stubs.expect_container_request_uuid + '\n')
 998
 999     @stubs
1000     def test_submit_priority(self, stubs):
1001         capture_stdout = cStringIO.StringIO()
1002         try:
1003             exited = arvados_cwl.main(
1004                 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1005                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1006                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1007             self.assertEqual(exited, 0)
1008         except:
1009             logging.exception("")
1010
1011         stubs.expect_container_spec["priority"] = 669
1012
1013         expect_container = copy.deepcopy(stubs.expect_container_spec)
1014         stubs.api.container_requests().create.assert_called_with(
1015             body=JsonDiffMatcher(expect_container))
1016         self.assertEqual(capture_stdout.getvalue(),
1017                          stubs.expect_container_request_uuid + '\n')
1018
1019
1020     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1021     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1022     @mock.patch("arvados.api")
1023     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1024         arvrunner = mock.MagicMock()
1025         arvrunner.project_uuid = ""
1026         api.return_value = mock.MagicMock()
1027         arvrunner.api = api.return_value
1028         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1029                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1030                                                                         "link_class": "docker_image_repo+tag",
1031                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1032                                                                         "owner_uuid": "",
1033                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1034                                                             {"items": [{"created_at": "",
1035                                                                         "head_uuid": "",
1036                                                                         "link_class": "docker_image_hash",
1037                                                                         "name": "123456",
1038                                                                         "owner_uuid": "",
1039                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1040         )
1041         find_one_image_hash.return_value = "123456"
1042
1043         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1044                                                                               "owner_uuid": "",
1045                                                                               "manifest_text": "",
1046                                                                               "properties": ""
1047                                                                           }], "items_available": 1, "offset": 0},)
1048         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1049         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
1050                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1051
1052     @stubs
1053     def test_submit_secrets(self, stubs):
1054         capture_stdout = cStringIO.StringIO()
1055         try:
1056             exited = arvados_cwl.main(
1057                 ["--submit", "--no-wait", "--api=containers", "--debug",
1058                  "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1059                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1060             self.assertEqual(exited, 0)
1061         except:
1062             logging.exception("")
1063
1064
1065         expect_container = {
1066             "command": [
1067                 "arvados-cwl-runner",
1068                 "--local",
1069                 "--api=containers",
1070                 "--no-log-timestamps",
1071                 "--disable-validate",
1072                 "--enable-reuse",
1073                 "--on-error=continue",
1074                 "--eval-timeout=20",
1075                 "/var/lib/cwl/workflow.json#main",
1076                 "/var/lib/cwl/cwl.input.json"
1077             ],
1078             "container_image": "arvados/jobs:"+arvados_cwl.__version__,
1079             "cwd": "/var/spool/cwl",
1080             "mounts": {
1081                 "/var/lib/cwl/cwl.input.json": {
1082                     "content": {
1083                         "pw": {
1084                             "$include": "/secrets/s0"
1085                         }
1086                     },
1087                     "kind": "json"
1088                 },
1089                 "/var/lib/cwl/workflow.json": {
1090                     "content": {
1091                         "$graph": [
1092                             {
1093                                 "$namespaces": {
1094                                     "cwltool": "http://commonwl.org/cwltool#"
1095                                 },
1096                                 "arguments": [
1097                                     "md5sum",
1098                                     "example.conf"
1099                                 ],
1100                                 "class": "CommandLineTool",
1101                                 "hints": [
1102                                     {
1103                                         "class": "http://commonwl.org/cwltool#Secrets",
1104                                         "secrets": [
1105                                             "#secret_job.cwl/pw"
1106                                         ]
1107                                     }
1108                                 ],
1109                                 "id": "#secret_job.cwl",
1110                                 "inputs": [
1111                                     {
1112                                         "id": "#secret_job.cwl/pw",
1113                                         "type": "string"
1114                                     }
1115                                 ],
1116                                 "outputs": [
1117                                     {
1118                                         "id": "#secret_job.cwl/out",
1119                                         "type": "stdout"
1120                                     }
1121                                 ],
1122                                 "stdout": "hashed_example.txt",
1123                                 "requirements": [
1124                                     {
1125                                         "class": "InitialWorkDirRequirement",
1126                                         "listing": [
1127                                             {
1128                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1129                                                 "entryname": "example.conf"
1130                                             }
1131                                         ]
1132                                     }
1133                                 ]
1134                             },
1135                             {
1136                                 "class": "Workflow",
1137                                 "hints": [
1138                                     {
1139                                         "class": "DockerRequirement",
1140                                         "dockerPull": "debian:8"
1141                                     },
1142                                     {
1143                                         "class": "http://commonwl.org/cwltool#Secrets",
1144                                         "secrets": [
1145                                             "#main/pw"
1146                                         ]
1147                                     }
1148                                 ],
1149                                 "id": "#main",
1150                                 "inputs": [
1151                                     {
1152                                         "id": "#main/pw",
1153                                         "type": "string"
1154                                     }
1155                                 ],
1156                                 "outputs": [
1157                                     {
1158                                         "id": "#main/out",
1159                                         "outputSource": "#main/step1/out",
1160                                         "type": "File"
1161                                     }
1162                                 ],
1163                                 "steps": [
1164                                     {
1165                                         "id": "#main/step1",
1166                                         "in": [
1167                                             {
1168                                                 "id": "#main/step1/pw",
1169                                                 "source": "#main/pw"
1170                                             }
1171                                         ],
1172                                         "out": [
1173                                             "#main/step1/out"
1174                                         ],
1175                                         "run": "#secret_job.cwl"
1176                                     }
1177                                 ]
1178                             }
1179                         ],
1180                         "cwlVersion": "v1.0"
1181                     },
1182                     "kind": "json"
1183                 },
1184                 "/var/spool/cwl": {
1185                     "kind": "collection",
1186                     "writable": True
1187                 },
1188                 "stdout": {
1189                     "kind": "file",
1190                     "path": "/var/spool/cwl/cwl.output.json"
1191                 }
1192             },
1193             "name": "secret_wf.cwl",
1194             "output_path": "/var/spool/cwl",
1195             "owner_uuid": None,
1196             "priority": 500,
1197             "properties": {},
1198             "runtime_constraints": {
1199                 "API": True,
1200                 "ram": 1073741824,
1201                 "vcpus": 1
1202             },
1203             "secret_mounts": {
1204                 "/secrets/s0": {
1205                     "content": "blorp",
1206                     "kind": "text"
1207                 }
1208             },
1209             "state": "Committed",
1210             "use_existing": True
1211         }
1212
1213         stubs.api.container_requests().create.assert_called_with(
1214             body=JsonDiffMatcher(expect_container))
1215         self.assertEqual(capture_stdout.getvalue(),
1216                          stubs.expect_container_request_uuid + '\n')
1217
1218
1219 class TestCreateTemplate(unittest.TestCase):
1220     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
1221
1222     def _adjust_script_params(self, expect_component):
1223         expect_component['script_parameters']['x'] = {
1224             'dataclass': 'File',
1225             'required': True,
1226             'type': 'File',
1227             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1228         }
1229         expect_component['script_parameters']['y'] = {
1230             'dataclass': 'Collection',
1231             'required': True,
1232             'type': 'Directory',
1233             'value': '99999999999999999999999999999998+99',
1234         }
1235         expect_component['script_parameters']['z'] = {
1236             'dataclass': 'Collection',
1237             'required': True,
1238             'type': 'Directory',
1239         }
1240
1241     @stubs
1242     def test_create(self, stubs):
1243         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1244
1245         capture_stdout = cStringIO.StringIO()
1246
1247         exited = arvados_cwl.main(
1248             ["--create-workflow", "--debug",
1249              "--api=jobs",
1250              "--project-uuid", project_uuid,
1251              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1252             capture_stdout, sys.stderr, api_client=stubs.api)
1253         self.assertEqual(exited, 0)
1254
1255         stubs.api.pipeline_instances().create.refute_called()
1256         stubs.api.jobs().create.refute_called()
1257
1258         expect_component = copy.deepcopy(stubs.expect_job_spec)
1259         self._adjust_script_params(expect_component)
1260         expect_template = {
1261             "components": {
1262                 "submit_wf.cwl": expect_component,
1263             },
1264             "name": "submit_wf.cwl",
1265             "owner_uuid": project_uuid,
1266         }
1267         stubs.api.pipeline_templates().create.assert_called_with(
1268             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1269
1270         self.assertEqual(capture_stdout.getvalue(),
1271                          stubs.expect_pipeline_template_uuid + '\n')
1272
1273
1274     @stubs
1275     def test_create_name(self, stubs):
1276         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1277
1278         capture_stdout = cStringIO.StringIO()
1279
1280         exited = arvados_cwl.main(
1281             ["--create-workflow", "--debug",
1282              "--project-uuid", project_uuid,
1283              "--api=jobs",
1284              "--name", "testing 123",
1285              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1286             capture_stdout, sys.stderr, api_client=stubs.api)
1287         self.assertEqual(exited, 0)
1288
1289         stubs.api.pipeline_instances().create.refute_called()
1290         stubs.api.jobs().create.refute_called()
1291
1292         expect_component = copy.deepcopy(stubs.expect_job_spec)
1293         self._adjust_script_params(expect_component)
1294         expect_template = {
1295             "components": {
1296                 "testing 123": expect_component,
1297             },
1298             "name": "testing 123",
1299             "owner_uuid": project_uuid,
1300         }
1301         stubs.api.pipeline_templates().create.assert_called_with(
1302             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1303
1304         self.assertEqual(capture_stdout.getvalue(),
1305                          stubs.expect_pipeline_template_uuid + '\n')
1306
1307
1308     @stubs
1309     def test_update_name(self, stubs):
1310         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1311
1312         capture_stdout = cStringIO.StringIO()
1313
1314         exited = arvados_cwl.main(
1315             ["--update-workflow", self.existing_template_uuid,
1316              "--debug",
1317              "--project-uuid", project_uuid,
1318              "--api=jobs",
1319              "--name", "testing 123",
1320              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1321             capture_stdout, sys.stderr, api_client=stubs.api)
1322         self.assertEqual(exited, 0)
1323
1324         stubs.api.pipeline_instances().create.refute_called()
1325         stubs.api.jobs().create.refute_called()
1326
1327         expect_component = copy.deepcopy(stubs.expect_job_spec)
1328         self._adjust_script_params(expect_component)
1329         expect_template = {
1330             "components": {
1331                 "testing 123": expect_component,
1332             },
1333             "name": "testing 123",
1334             "owner_uuid": project_uuid,
1335         }
1336         stubs.api.pipeline_templates().create.refute_called()
1337         stubs.api.pipeline_templates().update.assert_called_with(
1338             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1339
1340         self.assertEqual(capture_stdout.getvalue(),
1341                          self.existing_template_uuid + '\n')
1342
1343
1344 class TestCreateWorkflow(unittest.TestCase):
1345     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1346     expect_workflow = StripYAMLComments(
1347         open("tests/wf/expect_packed.cwl").read())
1348
1349     @stubs
1350     def test_create(self, stubs):
1351         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1352
1353         capture_stdout = cStringIO.StringIO()
1354
1355         exited = arvados_cwl.main(
1356             ["--create-workflow", "--debug",
1357              "--api=containers",
1358              "--project-uuid", project_uuid,
1359              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1360             capture_stdout, sys.stderr, api_client=stubs.api)
1361         self.assertEqual(exited, 0)
1362
1363         stubs.api.pipeline_templates().create.refute_called()
1364         stubs.api.container_requests().create.refute_called()
1365
1366         body = {
1367             "workflow": {
1368                 "owner_uuid": project_uuid,
1369                 "name": "submit_wf.cwl",
1370                 "description": "",
1371                 "definition": self.expect_workflow,
1372             }
1373         }
1374         stubs.api.workflows().create.assert_called_with(
1375             body=JsonDiffMatcher(body))
1376
1377         self.assertEqual(capture_stdout.getvalue(),
1378                          stubs.expect_workflow_uuid + '\n')
1379
1380
1381     @stubs
1382     def test_create_name(self, stubs):
1383         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1384
1385         capture_stdout = cStringIO.StringIO()
1386
1387         exited = arvados_cwl.main(
1388             ["--create-workflow", "--debug",
1389              "--api=containers",
1390              "--project-uuid", project_uuid,
1391              "--name", "testing 123",
1392              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1393             capture_stdout, sys.stderr, api_client=stubs.api)
1394         self.assertEqual(exited, 0)
1395
1396         stubs.api.pipeline_templates().create.refute_called()
1397         stubs.api.container_requests().create.refute_called()
1398
1399         body = {
1400             "workflow": {
1401                 "owner_uuid": project_uuid,
1402                 "name": "testing 123",
1403                 "description": "",
1404                 "definition": self.expect_workflow,
1405             }
1406         }
1407         stubs.api.workflows().create.assert_called_with(
1408             body=JsonDiffMatcher(body))
1409
1410         self.assertEqual(capture_stdout.getvalue(),
1411                          stubs.expect_workflow_uuid + '\n')
1412
1413     @stubs
1414     def test_incompatible_api(self, stubs):
1415         capture_stderr = cStringIO.StringIO()
1416         logging.getLogger('arvados.cwl-runner').addHandler(
1417             logging.StreamHandler(capture_stderr))
1418
1419         exited = arvados_cwl.main(
1420             ["--update-workflow", self.existing_workflow_uuid,
1421              "--api=jobs",
1422              "--debug",
1423              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1424             sys.stderr, sys.stderr, api_client=stubs.api)
1425         self.assertEqual(exited, 1)
1426         self.assertRegexpMatches(
1427             capture_stderr.getvalue(),
1428             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1429
1430     @stubs
1431     def test_update(self, stubs):
1432         capture_stdout = cStringIO.StringIO()
1433
1434         exited = arvados_cwl.main(
1435             ["--update-workflow", self.existing_workflow_uuid,
1436              "--debug",
1437              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1438             capture_stdout, sys.stderr, api_client=stubs.api)
1439         self.assertEqual(exited, 0)
1440
1441         body = {
1442             "workflow": {
1443                 "name": "submit_wf.cwl",
1444                 "description": "",
1445                 "definition": self.expect_workflow,
1446             }
1447         }
1448         stubs.api.workflows().update.assert_called_with(
1449             uuid=self.existing_workflow_uuid,
1450             body=JsonDiffMatcher(body))
1451         self.assertEqual(capture_stdout.getvalue(),
1452                          self.existing_workflow_uuid + '\n')
1453
1454
1455     @stubs
1456     def test_update_name(self, stubs):
1457         capture_stdout = cStringIO.StringIO()
1458
1459         exited = arvados_cwl.main(
1460             ["--update-workflow", self.existing_workflow_uuid,
1461              "--debug", "--name", "testing 123",
1462              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1463             capture_stdout, sys.stderr, api_client=stubs.api)
1464         self.assertEqual(exited, 0)
1465
1466         body = {
1467             "workflow": {
1468                 "name": "testing 123",
1469                 "description": "",
1470                 "definition": self.expect_workflow,
1471             }
1472         }
1473         stubs.api.workflows().update.assert_called_with(
1474             uuid=self.existing_workflow_uuid,
1475             body=JsonDiffMatcher(body))
1476         self.assertEqual(capture_stdout.getvalue(),
1477                          self.existing_workflow_uuid + '\n')
1478
1479
1480     @stubs
1481     def test_create_collection_per_tool(self, stubs):
1482         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1483
1484         capture_stdout = cStringIO.StringIO()
1485
1486         exited = arvados_cwl.main(
1487             ["--create-workflow", "--debug",
1488              "--api=containers",
1489              "--project-uuid", project_uuid,
1490              "tests/collection_per_tool/collection_per_tool.cwl"],
1491             capture_stdout, sys.stderr, api_client=stubs.api)
1492         self.assertEqual(exited, 0)
1493
1494         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1495         expect_workflow = StripYAMLComments(open(toolfile).read())
1496
1497         body = {
1498             "workflow": {
1499                 "owner_uuid": project_uuid,
1500                 "name": "collection_per_tool.cwl",
1501                 "description": "",
1502                 "definition": expect_workflow,
1503             }
1504         }
1505         stubs.api.workflows().create.assert_called_with(
1506             body=JsonDiffMatcher(body))
1507
1508         self.assertEqual(capture_stdout.getvalue(),
1509                          stubs.expect_workflow_uuid + '\n')
1510
1511 class TestTemplateInputs(unittest.TestCase):
1512     expect_template = {
1513         "components": {
1514             "inputs_test.cwl": {
1515                 'runtime_constraints': {
1516                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1517                     'min_ram_mb_per_node': 1024
1518                 },
1519                 'script_parameters': {
1520                     'cwl:tool':
1521                     '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1522                     'optionalFloatInput': None,
1523                     'fileInput': {
1524                         'type': 'File',
1525                         'dataclass': 'File',
1526                         'required': True,
1527                         'title': "It's a file; we expect to find some characters in it.",
1528                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1529                     },
1530                     'floatInput': {
1531                         'type': 'float',
1532                         'dataclass': 'number',
1533                         'required': True,
1534                         'title': 'Floats like a duck',
1535                         'default': 0.1,
1536                         'value': 0.1,
1537                     },
1538                     'optionalFloatInput': {
1539                         'type': ['null', 'float'],
1540                         'dataclass': 'number',
1541                         'required': False,
1542                     },
1543                     'boolInput': {
1544                         'type': 'boolean',
1545                         'dataclass': 'boolean',
1546                         'required': True,
1547                         'title': 'True or false?',
1548                     },
1549                 },
1550                 'repository': 'arvados',
1551                 'script_version': 'master',
1552                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1553                 'script': 'cwl-runner',
1554             },
1555         },
1556         "name": "inputs_test.cwl",
1557     }
1558
1559     @stubs
1560     def test_inputs_empty(self, stubs):
1561         exited = arvados_cwl.main(
1562             ["--create-template",
1563              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1564             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1565         self.assertEqual(exited, 0)
1566
1567         stubs.api.pipeline_templates().create.assert_called_with(
1568             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1569
1570     @stubs
1571     def test_inputs(self, stubs):
1572         exited = arvados_cwl.main(
1573             ["--create-template",
1574              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1575             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1576         self.assertEqual(exited, 0)
1577
1578         expect_template = copy.deepcopy(self.expect_template)
1579         params = expect_template[
1580             "components"]["inputs_test.cwl"]["script_parameters"]
1581         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1582         params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1583         params["floatInput"]["value"] = 1.234
1584         params["boolInput"]["value"] = True
1585
1586         stubs.api.pipeline_templates().create.assert_called_with(
1587             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)