sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20
  21 from io import BytesIO
  22
  23 # StringIO.StringIO and io.StringIO have different behavior write() is
  24 # called with both python2 (byte) strings and unicode strings
  25 # (specifically there's some logging in cwltool that causes trouble).
  26 # This isn't a problem on python3 because all string are unicode.
  27 if sys.version_info[0] < 3:
  28     from StringIO import StringIO
  29 else:
  30     from io import StringIO
  31
  32 import arvados
  33 import arvados.collection
  34 import arvados_cwl
  35 import arvados_cwl.executor
  36 import arvados_cwl.runner
  37 import arvados.keep
  38
  39 from .matcher import JsonDiffMatcher, StripYAMLComments
  40 from .mock_discovery import get_rootDesc
  41
  42 import ruamel.yaml as yaml
  43
  44 _rootDesc = None
  45
  46 def stubs(func):
  47     @functools.wraps(func)
  48     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  49     @mock.patch("arvados.collection.KeepClient")
  50     @mock.patch("arvados.keep.KeepClient")
  51     @mock.patch("arvados.events.subscribe")
  52     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  53         class Stubs(object):
  54             pass
  55         stubs = Stubs()
  56         stubs.events = events
  57         stubs.keepdocker = keepdocker
  58
  59         def putstub(p, **kwargs):
  60             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  61         keep_client1().put.side_effect = putstub
  62         keep_client1.put.side_effect = putstub
  63         keep_client2().put.side_effect = putstub
  64         keep_client2.put.side_effect = putstub
  65
  66         stubs.keep_client = keep_client2
  67         stubs.docker_images = {
  68             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  69             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  70             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  71             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  72         }
  73         def kd(a, b, image_name=None, image_tag=None):
  74             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  75         stubs.keepdocker.side_effect = kd
  76
  77         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  78         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  79
  80         if sys.version_info[0] < 3:
  81             stubs.capture_stdout = BytesIO()
  82         else:
  83             stubs.capture_stdout = StringIO()
  84
  85         stubs.api = mock.MagicMock()
  86         stubs.api._rootDesc = get_rootDesc()
  87         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  88
  89         stubs.api.users().current().execute.return_value = {
  90             "uuid": stubs.fake_user_uuid,
  91         }
  92         stubs.api.collections().list().execute.return_value = {"items": []}
  93         stubs.api.containers().current().execute.return_value = {
  94             "uuid": stubs.fake_container_uuid,
  95         }
  96
  97         class CollectionExecute(object):
  98             def __init__(self, exe):
  99                 self.exe = exe
 100             def execute(self, num_retries=None):
 101                 return self.exe
 102
 103         def collection_createstub(created_collections, body, ensure_unique_name=None):
 104             mt = body["manifest_text"].encode('utf-8')
 105             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 106             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 107             created_collections[uuid] = {
 108                 "uuid": uuid,
 109                 "portable_data_hash": pdh,
 110                 "manifest_text": mt.decode('utf-8')
 111             }
 112             return CollectionExecute(created_collections[uuid])
 113
 114         def collection_getstub(created_collections, uuid):
 115             for v in viewvalues(created_collections):
 116                 if uuid in (v["uuid"], v["portable_data_hash"]):
 117                     return CollectionExecute(v)
 118
 119         created_collections = {
 120             "99999999999999999999999999999998+99": {
 121                 "uuid": "",
 122                 "portable_data_hash": "99999999999999999999999999999998+99",
 123                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 124             },
 125             "99999999999999999999999999999997+99": {
 126                 "uuid": "",
 127                 "portable_data_hash": "99999999999999999999999999999997+99",
 128                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 129             },
 130             "99999999999999999999999999999994+99": {
 131                 "uuid": "",
 132                 "portable_data_hash": "99999999999999999999999999999994+99",
 133                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 134             },
 135             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 136                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 137                 "portable_data_hash": "999999999999999999999999999999d3+99",
 138                 "manifest_text": ""
 139             },
 140             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 141                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 142                 "portable_data_hash": "999999999999999999999999999999d4+99",
 143                 "manifest_text": ""
 144             },
 145             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 146                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 147                 "portable_data_hash": "999999999999999999999999999999d5+99",
 148                 "manifest_text": ""
 149             },
 150             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 151                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 152                 "portable_data_hash": "999999999999999999999999999999d6+99",
 153                 "manifest_text": ""
 154             }
 155         }
 156         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 157         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 158
 159         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 160         stubs.api.jobs().create().execute.return_value = {
 161             "uuid": stubs.expect_job_uuid,
 162             "state": "Queued",
 163         }
 164
 165         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 166         stubs.api.container_requests().create().execute.return_value = {
 167             "uuid": stubs.expect_container_request_uuid,
 168             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 169             "state": "Queued"
 170         }
 171
 172         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 173         stubs.api.pipeline_templates().create().execute.return_value = {
 174             "uuid": stubs.expect_pipeline_template_uuid,
 175         }
 176         stubs.expect_job_spec = {
 177             'runtime_constraints': {
 178                 'docker_image': '999999999999999999999999999999d3+99',
 179                 'min_ram_mb_per_node': 1024
 180             },
 181             'script_parameters': {
 182                 'x': {
 183                     'basename': 'blorp.txt',
 184                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 185                     'class': 'File'
 186                 },
 187                 'y': {
 188                     'basename': '99999999999999999999999999999998+99',
 189                     'location': 'keep:99999999999999999999999999999998+99',
 190                     'class': 'Directory'
 191                 },
 192                 'z': {
 193                     'basename': 'anonymous',
 194                     "listing": [{
 195                         "basename": "renamed.txt",
 196                         "class": "File",
 197                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 198                         "size": 0
 199                     }],
 200                     'class': 'Directory'
 201                 },
 202                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 203             },
 204             'repository': 'arvados',
 205             'script_version': 'master',
 206             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 207             'script': 'cwl-runner'
 208         }
 209         stubs.pipeline_component = stubs.expect_job_spec.copy()
 210         stubs.expect_pipeline_instance = {
 211             'name': 'submit_wf.cwl',
 212             'state': 'RunningOnServer',
 213             'owner_uuid': None,
 214             "components": {
 215                 "cwl-runner": {
 216                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 217                     'script_parameters': {
 218                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 219                         'x': {"value": {
 220                             'basename': 'blorp.txt',
 221                             'class': 'File',
 222                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 223                             "size": 16
 224                         }},
 225                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 226                               'listing': [
 227                                   {
 228                                       'basename': 'renamed.txt',
 229                                       'class': 'File', 'location':
 230                                       'keep:99999999999999999999999999999998+99/file1.txt',
 231                                       'size': 0
 232                                   }
 233                               ]}},
 234                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 235                         'arv:debug': True,
 236                         'arv:enable_reuse': True,
 237                         'arv:on_error': 'continue'
 238                     },
 239                     'repository': 'arvados',
 240                     'script_version': 'master',
 241                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 242                     'script': 'cwl-runner',
 243                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 244                 }
 245             }
 246         }
 247         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 248         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 249         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 250         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 251         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 252             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 253             "state": "Queued"
 254         }
 255         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 256         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 257
 258         with open("tests/wf/submit_wf_packed.cwl") as f:
 259             expect_packed_workflow = yaml.round_trip_load(f)
 260
 261         stubs.expect_container_spec = {
 262             'priority': 500,
 263             'mounts': {
 264                 '/var/spool/cwl': {
 265                     'writable': True,
 266                     'kind': 'collection'
 267                 },
 268                 '/var/lib/cwl/workflow.json': {
 269                     'content': expect_packed_workflow,
 270                     'kind': 'json'
 271                 },
 272                 'stdout': {
 273                     'path': '/var/spool/cwl/cwl.output.json',
 274                     'kind': 'file'
 275                 },
 276                 '/var/lib/cwl/cwl.input.json': {
 277                     'kind': 'json',
 278                     'content': {
 279                         'y': {
 280                             'basename': '99999999999999999999999999999998+99',
 281                             'location': 'keep:99999999999999999999999999999998+99',
 282                             'class': 'Directory'},
 283                         'x': {
 284                             'basename': u'blorp.txt',
 285                             'class': 'File',
 286                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 287                             "size": 16
 288                         },
 289                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 290                             {'basename': 'renamed.txt',
 291                              'class': 'File',
 292                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 293                              'size': 0
 294                             }
 295                         ]}
 296                     },
 297                     'kind': 'json'
 298                 }
 299             },
 300             'secret_mounts': {},
 301             'state': 'Committed',
 302             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 303                         '--no-log-timestamps', '--disable-validate',
 304                         '--eval-timeout=20', '--thread-count=1',
 305                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 306                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 307             'name': 'submit_wf.cwl',
 308             'container_image': '999999999999999999999999999999d3+99',
 309             'output_path': '/var/spool/cwl',
 310             'cwd': '/var/spool/cwl',
 311             'runtime_constraints': {
 312                 'API': True,
 313                 'vcpus': 1,
 314                 'ram': (1024+256)*1024*1024
 315             },
 316             'use_existing': True,
 317             'properties': {},
 318             'secret_mounts': {}
 319         }
 320
 321         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 322         stubs.api.workflows().create().execute.return_value = {
 323             "uuid": stubs.expect_workflow_uuid,
 324         }
 325         def update_mock(**kwargs):
 326             stubs.updated_uuid = kwargs.get('uuid')
 327             return mock.DEFAULT
 328         stubs.api.workflows().update.side_effect = update_mock
 329         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 330             "uuid": stubs.updated_uuid,
 331         }
 332
 333         return func(self, stubs, *args, **kwargs)
 334     return wrapped
 335
 336
 337 class TestSubmit(unittest.TestCase):
 338     @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
 339     @mock.patch("time.sleep")
 340     @stubs
 341     def test_submit(self, stubs, tm, arvdock):
 342         def get_image(api_client, dockerRequirement, pull_image, project_uuid):
 343             if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
 344                 return '999999999999999999999999999999d3+99'
 345             elif dockerRequirement["dockerPull"] == "debian:8":
 346                 return '999999999999999999999999999999d4+99'
 347         arvdock.side_effect = get_image
 348
 349         exited = arvados_cwl.main(
 350             ["--submit", "--no-wait", "--api=jobs", "--debug",
 351              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 352             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 353
 354         stubs.api.collections().create.assert_has_calls([
 355             mock.call(body=JsonDiffMatcher({
 356                 'manifest_text':
 357                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 358                 'replication_desired': None,
 359                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 360             }), ensure_unique_name=False),
 361             mock.call(body=JsonDiffMatcher({
 362                 'manifest_text':
 363                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 364                 'replication_desired': None,
 365                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 366             }), ensure_unique_name=False),
 367             mock.call(body=JsonDiffMatcher({
 368                 'manifest_text':
 369                 ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
 370                 'replication_desired': None,
 371                 'name': 'submit_wf.cwl',
 372             }), ensure_unique_name=True)        ])
 373
 374         arvdock.assert_has_calls([
 375             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
 376             mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
 377             mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
 378         ])
 379
 380         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 381         stubs.api.pipeline_instances().create.assert_called_with(
 382             body=JsonDiffMatcher(expect_pipeline))
 383         self.assertEqual(stubs.capture_stdout.getvalue(),
 384                          stubs.expect_pipeline_uuid + '\n')
 385         self.assertEqual(exited, 0)
 386
 387     @mock.patch("time.sleep")
 388     @stubs
 389     def test_submit_no_reuse(self, stubs, tm):
 390         exited = arvados_cwl.main(
 391             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
 392              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 393             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 394
 395         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 396         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 397         expect_pipeline["properties"] = {"run_options": {"enable_job_reuse": False}}
 398
 399         stubs.api.pipeline_instances().create.assert_called_with(
 400             body=JsonDiffMatcher(expect_pipeline))
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_pipeline_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_error_when_multiple_storage_classes_specified(self, stubs):
 407         storage_classes = "foo,bar"
 408         exited = arvados_cwl.main(
 409                 ["--debug", "--storage-classes", storage_classes,
 410                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 411                 sys.stdin, sys.stderr, api_client=stubs.api)
 412         self.assertEqual(exited, 1)
 413
 414     @mock.patch("time.sleep")
 415     @stubs
 416     def test_submit_on_error(self, stubs, tm):
 417         exited = arvados_cwl.main(
 418             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
 419              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 420             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 421
 422         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 423         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 424
 425         stubs.api.pipeline_instances().create.assert_called_with(
 426             body=JsonDiffMatcher(expect_pipeline))
 427         self.assertEqual(stubs.capture_stdout.getvalue(),
 428                          stubs.expect_pipeline_uuid + '\n')
 429         self.assertEqual(exited, 0)
 430
 431     @mock.patch("time.sleep")
 432     @stubs
 433     def test_submit_runner_ram(self, stubs, tm):
 434         exited = arvados_cwl.main(
 435             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 436              "--api=jobs",
 437              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 438             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 439
 440         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 441         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 442
 443         stubs.api.pipeline_instances().create.assert_called_with(
 444             body=JsonDiffMatcher(expect_pipeline))
 445         self.assertEqual(stubs.capture_stdout.getvalue(),
 446                          stubs.expect_pipeline_uuid + '\n')
 447         self.assertEqual(exited, 0)
 448
 449     @mock.patch("time.sleep")
 450     @stubs
 451     def test_submit_invalid_runner_ram(self, stubs, tm):
 452         exited = arvados_cwl.main(
 453             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 454              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 455             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 456         self.assertEqual(exited, 1)
 457
 458     @mock.patch("time.sleep")
 459     @stubs
 460     def test_submit_output_name(self, stubs, tm):
 461         output_name = "test_output_name"
 462
 463         exited = arvados_cwl.main(
 464             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 465              "--api=jobs",
 466              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 468
 469         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 470         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 471
 472         stubs.api.pipeline_instances().create.assert_called_with(
 473             body=JsonDiffMatcher(expect_pipeline))
 474         self.assertEqual(stubs.capture_stdout.getvalue(),
 475                          stubs.expect_pipeline_uuid + '\n')
 476         self.assertEqual(exited, 0)
 477
 478     @mock.patch("time.sleep")
 479     @stubs
 480     def test_submit_pipeline_name(self, stubs, tm):
 481         exited = arvados_cwl.main(
 482             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 483              "--api=jobs",
 484              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 485             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 486         self.assertEqual(exited, 0)
 487
 488         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 489         expect_pipeline["name"] = "hello job 123"
 490
 491         stubs.api.pipeline_instances().create.assert_called_with(
 492             body=JsonDiffMatcher(expect_pipeline))
 493         self.assertEqual(stubs.capture_stdout.getvalue(),
 494                          stubs.expect_pipeline_uuid + '\n')
 495
 496     @mock.patch("time.sleep")
 497     @stubs
 498     def test_submit_output_tags(self, stubs, tm):
 499         output_tags = "tag0,tag1,tag2"
 500
 501         exited = arvados_cwl.main(
 502             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 503              "--api=jobs",
 504              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 505             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 506         self.assertEqual(exited, 0)
 507
 508         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 509         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 510
 511         stubs.api.pipeline_instances().create.assert_called_with(
 512             body=JsonDiffMatcher(expect_pipeline))
 513         self.assertEqual(stubs.capture_stdout.getvalue(),
 514                          stubs.expect_pipeline_uuid + '\n')
 515
 516     @mock.patch("time.sleep")
 517     @stubs
 518     def test_submit_with_project_uuid(self, stubs, tm):
 519         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 520
 521         exited = arvados_cwl.main(
 522             ["--submit", "--no-wait", "--debug",
 523              "--project-uuid", project_uuid,
 524              "--api=jobs",
 525              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 526             sys.stdout, sys.stderr, api_client=stubs.api)
 527         self.assertEqual(exited, 0)
 528
 529         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 530         expect_pipeline["owner_uuid"] = project_uuid
 531         stubs.api.pipeline_instances().create.assert_called_with(
 532             body=JsonDiffMatcher(expect_pipeline))
 533
 534     @stubs
 535     def test_submit_container(self, stubs):
 536         exited = arvados_cwl.main(
 537             ["--submit", "--no-wait", "--api=containers", "--debug",
 538                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 539             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 540
 541         stubs.api.collections().create.assert_has_calls([
 542             mock.call(body=JsonDiffMatcher({
 543                 'manifest_text':
 544                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 545                 'replication_desired': None,
 546                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 547             }), ensure_unique_name=False),
 548             mock.call(body=JsonDiffMatcher({
 549                 'manifest_text':
 550                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 551                 'replication_desired': None,
 552                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 553             }), ensure_unique_name=False),
 554             ])
 555
 556         expect_container = copy.deepcopy(stubs.expect_container_spec)
 557         stubs.api.container_requests().create.assert_called_with(
 558             body=JsonDiffMatcher(expect_container))
 559         self.assertEqual(stubs.capture_stdout.getvalue(),
 560                          stubs.expect_container_request_uuid + '\n')
 561         self.assertEqual(exited, 0)
 562
 563     @stubs
 564     def test_submit_container_no_reuse(self, stubs):
 565         exited = arvados_cwl.main(
 566             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 567                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 568             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 569
 570         expect_container = copy.deepcopy(stubs.expect_container_spec)
 571         expect_container["command"] = [
 572             'arvados-cwl-runner', '--local', '--api=containers',
 573             '--no-log-timestamps', '--disable-validate',
 574             '--eval-timeout=20', '--thread-count=1',
 575             '--disable-reuse', "--collection-cache-size=256",
 576             '--debug', '--on-error=continue',
 577             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 578         expect_container["use_existing"] = False
 579
 580         stubs.api.container_requests().create.assert_called_with(
 581             body=JsonDiffMatcher(expect_container))
 582         self.assertEqual(stubs.capture_stdout.getvalue(),
 583                          stubs.expect_container_request_uuid + '\n')
 584         self.assertEqual(exited, 0)
 585
 586     @stubs
 587     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 588         exited = arvados_cwl.main(
 589             ["--submit", "--no-wait", "--api=containers", "--debug",
 590              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 591             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 592         self.assertEqual(exited, 0)
 593
 594         expect_container = copy.deepcopy(stubs.expect_container_spec)
 595         expect_container["command"] = [
 596             'arvados-cwl-runner', '--local', '--api=containers',
 597             '--no-log-timestamps', '--disable-validate',
 598             '--eval-timeout=20', '--thread-count=1',
 599             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 600             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 601         expect_container["use_existing"] = False
 602         expect_container["name"] = "submit_wf_no_reuse.cwl"
 603         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 604             {
 605                 "class": "http://arvados.org/cwl#ReuseRequirement",
 606                 "enableReuse": False,
 607             },
 608         ]
 609         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 610             "arv": "http://arvados.org/cwl#",
 611             "cwltool": "http://commonwl.org/cwltool#"
 612         }
 613
 614         stubs.api.container_requests().create.assert_called_with(
 615             body=JsonDiffMatcher(expect_container))
 616         self.assertEqual(stubs.capture_stdout.getvalue(),
 617                          stubs.expect_container_request_uuid + '\n')
 618
 619
 620     @stubs
 621     def test_submit_container_on_error(self, stubs):
 622         exited = arvados_cwl.main(
 623             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 624                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 625             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 626
 627         expect_container = copy.deepcopy(stubs.expect_container_spec)
 628         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 629                                        '--no-log-timestamps', '--disable-validate',
 630                                        '--eval-timeout=20', '--thread-count=1',
 631                                        '--enable-reuse', "--collection-cache-size=256",
 632                                        '--debug', '--on-error=stop',
 633                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 634
 635         stubs.api.container_requests().create.assert_called_with(
 636             body=JsonDiffMatcher(expect_container))
 637         self.assertEqual(stubs.capture_stdout.getvalue(),
 638                          stubs.expect_container_request_uuid + '\n')
 639         self.assertEqual(exited, 0)
 640
 641     @stubs
 642     def test_submit_container_output_name(self, stubs):
 643         output_name = "test_output_name"
 644
 645         exited = arvados_cwl.main(
 646             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 647                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 648             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 649
 650         expect_container = copy.deepcopy(stubs.expect_container_spec)
 651         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 652                                        '--no-log-timestamps', '--disable-validate',
 653                                        '--eval-timeout=20', '--thread-count=1',
 654                                        '--enable-reuse', "--collection-cache-size=256",
 655                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 656                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 657         expect_container["output_name"] = output_name
 658
 659         stubs.api.container_requests().create.assert_called_with(
 660             body=JsonDiffMatcher(expect_container))
 661         self.assertEqual(stubs.capture_stdout.getvalue(),
 662                          stubs.expect_container_request_uuid + '\n')
 663         self.assertEqual(exited, 0)
 664
 665     @stubs
 666     def test_submit_storage_classes(self, stubs):
 667         exited = arvados_cwl.main(
 668             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 669                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 670             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 671
 672         expect_container = copy.deepcopy(stubs.expect_container_spec)
 673         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 674                                        '--no-log-timestamps', '--disable-validate',
 675                                        '--eval-timeout=20', '--thread-count=1',
 676                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 677                                        "--storage-classes=foo", '--on-error=continue',
 678                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 679
 680         stubs.api.container_requests().create.assert_called_with(
 681             body=JsonDiffMatcher(expect_container))
 682         self.assertEqual(stubs.capture_stdout.getvalue(),
 683                          stubs.expect_container_request_uuid + '\n')
 684         self.assertEqual(exited, 0)
 685
 686     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 687     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 688     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 689     @stubs
 690     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 691         def set_final_output(job_order, output_callback, runtimeContext):
 692             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 693             return []
 694         job.side_effect = set_final_output
 695
 696         exited = arvados_cwl.main(
 697             ["--debug", "--local", "--storage-classes=foo",
 698                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 699             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 700
 701         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 702         self.assertEqual(exited, 0)
 703
 704     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 705     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 706     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
 707     @stubs
 708     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 709         def set_final_output(job_order, output_callback, runtimeContext):
 710             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 711             return []
 712         job.side_effect = set_final_output
 713
 714         exited = arvados_cwl.main(
 715             ["--debug", "--local",
 716                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 717             sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 718
 719         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 720         self.assertEqual(exited, 0)
 721
 722     @stubs
 723     def test_submit_container_output_ttl(self, stubs):
 724         exited = arvados_cwl.main(
 725             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 726                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 727             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 728
 729         expect_container = copy.deepcopy(stubs.expect_container_spec)
 730         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 731                                        '--no-log-timestamps', '--disable-validate',
 732                                        '--eval-timeout=20', '--thread-count=1',
 733                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 734                                        '--on-error=continue',
 735                                        "--intermediate-output-ttl=3600",
 736                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 737
 738         stubs.api.container_requests().create.assert_called_with(
 739             body=JsonDiffMatcher(expect_container))
 740         self.assertEqual(stubs.capture_stdout.getvalue(),
 741                          stubs.expect_container_request_uuid + '\n')
 742         self.assertEqual(exited, 0)
 743
 744     @stubs
 745     def test_submit_container_trash_intermediate(self, stubs):
 746         exited = arvados_cwl.main(
 747             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 748                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 749             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 750
 751
 752         expect_container = copy.deepcopy(stubs.expect_container_spec)
 753         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 754                                        '--no-log-timestamps', '--disable-validate',
 755                                        '--eval-timeout=20', '--thread-count=1',
 756                                        '--enable-reuse', "--collection-cache-size=256",
 757                                        '--debug', '--on-error=continue',
 758                                        "--trash-intermediate",
 759                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 760
 761         stubs.api.container_requests().create.assert_called_with(
 762             body=JsonDiffMatcher(expect_container))
 763         self.assertEqual(stubs.capture_stdout.getvalue(),
 764                          stubs.expect_container_request_uuid + '\n')
 765         self.assertEqual(exited, 0)
 766
 767     @stubs
 768     def test_submit_container_output_tags(self, stubs):
 769         output_tags = "tag0,tag1,tag2"
 770
 771         exited = arvados_cwl.main(
 772             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 773                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 774             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 775
 776         expect_container = copy.deepcopy(stubs.expect_container_spec)
 777         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 778                                        '--no-log-timestamps', '--disable-validate',
 779                                        '--eval-timeout=20', '--thread-count=1',
 780                                        '--enable-reuse', "--collection-cache-size=256",
 781                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 782                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 783
 784         stubs.api.container_requests().create.assert_called_with(
 785             body=JsonDiffMatcher(expect_container))
 786         self.assertEqual(stubs.capture_stdout.getvalue(),
 787                          stubs.expect_container_request_uuid + '\n')
 788         self.assertEqual(exited, 0)
 789
 790     @stubs
 791     def test_submit_container_runner_ram(self, stubs):
 792         exited = arvados_cwl.main(
 793             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 794                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 795             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 796
 797         expect_container = copy.deepcopy(stubs.expect_container_spec)
 798         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 799
 800         stubs.api.container_requests().create.assert_called_with(
 801             body=JsonDiffMatcher(expect_container))
 802         self.assertEqual(stubs.capture_stdout.getvalue(),
 803                          stubs.expect_container_request_uuid + '\n')
 804         self.assertEqual(exited, 0)
 805
 806     @mock.patch("arvados.collection.CollectionReader")
 807     @mock.patch("time.sleep")
 808     @stubs
 809     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 810         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 811         exited = arvados_cwl.main(
 812             ["--submit", "--no-wait", "--api=containers", "--debug",
 813              "tests/wf/submit_keepref_wf.cwl"],
 814             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 815         self.assertEqual(exited, 0)
 816
 817     @mock.patch("arvados.collection.CollectionReader")
 818     @mock.patch("time.sleep")
 819     @stubs
 820     def test_submit_keepref(self, stubs, tm, reader):
 821         with open("tests/wf/expect_arvworkflow.cwl") as f:
 822             reader().open().__enter__().read.return_value = f.read()
 823
 824         exited = arvados_cwl.main(
 825             ["--submit", "--no-wait", "--api=containers", "--debug",
 826              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 827             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 828
 829         expect_container = {
 830             'priority': 500,
 831             'mounts': {
 832                 '/var/spool/cwl': {
 833                     'writable': True,
 834                     'kind': 'collection'
 835                 },
 836                 'stdout': {
 837                     'path': '/var/spool/cwl/cwl.output.json',
 838                     'kind': 'file'
 839                 },
 840                 '/var/lib/cwl/workflow': {
 841                     'portable_data_hash': '99999999999999999999999999999994+99',
 842                     'kind': 'collection'
 843                 },
 844                 '/var/lib/cwl/cwl.input.json': {
 845                     'content': {
 846                         'x': 'XxX'
 847                     },
 848                     'kind': 'json'
 849                 }
 850             }, 'state': 'Committed',
 851             'output_path': '/var/spool/cwl',
 852             'name': 'expect_arvworkflow.cwl#main',
 853             'container_image': '999999999999999999999999999999d3+99',
 854             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 855                         '--no-log-timestamps', '--disable-validate',
 856                         '--eval-timeout=20', '--thread-count=1',
 857                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 858                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 859             'cwd': '/var/spool/cwl',
 860             'runtime_constraints': {
 861                 'API': True,
 862                 'vcpus': 1,
 863                 'ram': 1342177280
 864             },
 865             'use_existing': True,
 866             'properties': {},
 867             'secret_mounts': {}
 868         }
 869
 870         stubs.api.container_requests().create.assert_called_with(
 871             body=JsonDiffMatcher(expect_container))
 872         self.assertEqual(stubs.capture_stdout.getvalue(),
 873                          stubs.expect_container_request_uuid + '\n')
 874         self.assertEqual(exited, 0)
 875
 876     @mock.patch("arvados.collection.CollectionReader")
 877     @mock.patch("time.sleep")
 878     @stubs
 879     def test_submit_jobs_keepref(self, stubs, tm, reader):
 880         with open("tests/wf/expect_arvworkflow.cwl") as f:
 881             reader().open().__enter__().read.return_value = f.read()
 882
 883         exited = arvados_cwl.main(
 884             ["--submit", "--no-wait", "--api=jobs", "--debug",
 885              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 886             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 887
 888         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 889         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
 890         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
 891         del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
 892         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
 893         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
 894         stubs.api.pipeline_instances().create.assert_called_with(
 895             body=JsonDiffMatcher(expect_pipeline))
 896         self.assertEqual(exited, 0)
 897
 898     @mock.patch("time.sleep")
 899     @stubs
 900     def test_submit_arvworkflow(self, stubs, tm):
 901         with open("tests/wf/expect_arvworkflow.cwl") as f:
 902             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 903
 904         exited = arvados_cwl.main(
 905             ["--submit", "--no-wait", "--api=containers", "--debug",
 906              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 907             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 908
 909         expect_container = {
 910             'priority': 500,
 911             'mounts': {
 912                 '/var/spool/cwl': {
 913                     'writable': True,
 914                     'kind': 'collection'
 915                 },
 916                 'stdout': {
 917                     'path': '/var/spool/cwl/cwl.output.json',
 918                     'kind': 'file'
 919                 },
 920                 '/var/lib/cwl/workflow.json': {
 921                     'kind': 'json',
 922                     'content': {
 923                         'cwlVersion': 'v1.0',
 924                         '$graph': [
 925                             {
 926                                 'id': '#main',
 927                                 'inputs': [
 928                                     {'type': 'string', 'id': '#main/x'}
 929                                 ],
 930                                 'steps': [
 931                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 932                                      'run': '#submit_tool.cwl',
 933                                      'id': '#main/step1',
 934                                      'out': []}
 935                                 ],
 936                                 'class': 'Workflow',
 937                                 'outputs': []
 938                             },
 939                             {
 940                                 'inputs': [
 941                                     {
 942                                         'inputBinding': {'position': 1},
 943                                         'type': 'string',
 944                                         'id': '#submit_tool.cwl/x'}
 945                                 ],
 946                                 'requirements': [
 947                                     {
 948                                         'dockerPull': 'debian:8',
 949                                         'class': 'DockerRequirement',
 950                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 951                                     }
 952                                 ],
 953                                 'id': '#submit_tool.cwl',
 954                                 'outputs': [],
 955                                 'baseCommand': 'cat',
 956                                 'class': 'CommandLineTool'
 957                             }
 958                         ]
 959                     }
 960                 },
 961                 '/var/lib/cwl/cwl.input.json': {
 962                     'content': {
 963                         'x': 'XxX'
 964                     },
 965                     'kind': 'json'
 966                 }
 967             }, 'state': 'Committed',
 968             'output_path': '/var/spool/cwl',
 969             'name': 'a test workflow',
 970             'container_image': "999999999999999999999999999999d3+99",
 971             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 972                         '--no-log-timestamps', '--disable-validate',
 973                         '--eval-timeout=20', '--thread-count=1',
 974                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 975                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 976             'cwd': '/var/spool/cwl',
 977             'runtime_constraints': {
 978                 'API': True,
 979                 'vcpus': 1,
 980                 'ram': 1342177280
 981             },
 982             'use_existing': True,
 983             'properties': {
 984                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 985             },
 986             'secret_mounts': {}
 987         }
 988
 989         stubs.api.container_requests().create.assert_called_with(
 990             body=JsonDiffMatcher(expect_container))
 991         self.assertEqual(stubs.capture_stdout.getvalue(),
 992                          stubs.expect_container_request_uuid + '\n')
 993         self.assertEqual(exited, 0)
 994
 995     @stubs
 996     def test_submit_container_name(self, stubs):
 997         exited = arvados_cwl.main(
 998             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 999                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1000             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1001
1002         expect_container = copy.deepcopy(stubs.expect_container_spec)
1003         expect_container["name"] = "hello container 123"
1004
1005         stubs.api.container_requests().create.assert_called_with(
1006             body=JsonDiffMatcher(expect_container))
1007         self.assertEqual(stubs.capture_stdout.getvalue(),
1008                          stubs.expect_container_request_uuid + '\n')
1009         self.assertEqual(exited, 0)
1010
1011     @stubs
1012     def test_submit_missing_input(self, stubs):
1013         exited = arvados_cwl.main(
1014             ["--submit", "--no-wait", "--api=containers", "--debug",
1015              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1016             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1017         self.assertEqual(exited, 0)
1018
1019         exited = arvados_cwl.main(
1020             ["--submit", "--no-wait", "--api=containers", "--debug",
1021              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
1022             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1023         self.assertEqual(exited, 1)
1024
1025     @stubs
1026     def test_submit_container_project(self, stubs):
1027         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1028         exited = arvados_cwl.main(
1029             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
1030                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1031             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1032
1033         expect_container = copy.deepcopy(stubs.expect_container_spec)
1034         expect_container["owner_uuid"] = project_uuid
1035         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1036                                        '--no-log-timestamps', '--disable-validate',
1037                                        "--eval-timeout=20", "--thread-count=1",
1038                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
1039                                        '--on-error=continue',
1040                                        '--project-uuid='+project_uuid,
1041                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1042
1043         stubs.api.container_requests().create.assert_called_with(
1044             body=JsonDiffMatcher(expect_container))
1045         self.assertEqual(stubs.capture_stdout.getvalue(),
1046                          stubs.expect_container_request_uuid + '\n')
1047         self.assertEqual(exited, 0)
1048
1049     @stubs
1050     def test_submit_container_eval_timeout(self, stubs):
1051         exited = arvados_cwl.main(
1052             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
1053                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1054             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1055
1056         expect_container = copy.deepcopy(stubs.expect_container_spec)
1057         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1058                                        '--no-log-timestamps', '--disable-validate',
1059                                        '--eval-timeout=60.0', '--thread-count=1',
1060                                        '--enable-reuse', "--collection-cache-size=256",
1061                                        '--debug', '--on-error=continue',
1062                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1063
1064         stubs.api.container_requests().create.assert_called_with(
1065             body=JsonDiffMatcher(expect_container))
1066         self.assertEqual(stubs.capture_stdout.getvalue(),
1067                          stubs.expect_container_request_uuid + '\n')
1068         self.assertEqual(exited, 0)
1069
1070     @stubs
1071     def test_submit_container_collection_cache(self, stubs):
1072         exited = arvados_cwl.main(
1073             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
1074                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1075             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1076
1077         expect_container = copy.deepcopy(stubs.expect_container_spec)
1078         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1079                                        '--no-log-timestamps', '--disable-validate',
1080                                        '--eval-timeout=20', '--thread-count=1',
1081                                        '--enable-reuse', "--collection-cache-size=500",
1082                                        '--debug', '--on-error=continue',
1083                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1084         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1085
1086         stubs.api.container_requests().create.assert_called_with(
1087             body=JsonDiffMatcher(expect_container))
1088         self.assertEqual(stubs.capture_stdout.getvalue(),
1089                          stubs.expect_container_request_uuid + '\n')
1090         self.assertEqual(exited, 0)
1091
1092     @stubs
1093     def test_submit_container_thread_count(self, stubs):
1094         exited = arvados_cwl.main(
1095             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1096                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1097             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1098
1099         expect_container = copy.deepcopy(stubs.expect_container_spec)
1100         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1101                                        '--no-log-timestamps', '--disable-validate',
1102                                        '--eval-timeout=20', '--thread-count=20',
1103                                        '--enable-reuse', "--collection-cache-size=256",
1104                                        '--debug', '--on-error=continue',
1105                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1106
1107         stubs.api.container_requests().create.assert_called_with(
1108             body=JsonDiffMatcher(expect_container))
1109         self.assertEqual(stubs.capture_stdout.getvalue(),
1110                          stubs.expect_container_request_uuid + '\n')
1111         self.assertEqual(exited, 0)
1112
1113     @stubs
1114     def test_submit_job_runner_image(self, stubs):
1115         exited = arvados_cwl.main(
1116             ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
1117                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1118             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1119
1120         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
1121
1122         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
1123         stubs.api.pipeline_instances().create.assert_called_with(
1124             body=JsonDiffMatcher(expect_pipeline))
1125         self.assertEqual(stubs.capture_stdout.getvalue(),
1126                          stubs.expect_pipeline_uuid + '\n')
1127         self.assertEqual(exited, 0)
1128
1129     @stubs
1130     def test_submit_container_runner_image(self, stubs):
1131         exited = arvados_cwl.main(
1132             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1133                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1134             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1135
1136         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1137
1138         expect_container = copy.deepcopy(stubs.expect_container_spec)
1139         stubs.api.container_requests().create.assert_called_with(
1140             body=JsonDiffMatcher(expect_container))
1141         self.assertEqual(stubs.capture_stdout.getvalue(),
1142                          stubs.expect_container_request_uuid + '\n')
1143         self.assertEqual(exited, 0)
1144
1145     @stubs
1146     def test_submit_priority(self, stubs):
1147         exited = arvados_cwl.main(
1148             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1149                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1150             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1151
1152         stubs.expect_container_spec["priority"] = 669
1153
1154         expect_container = copy.deepcopy(stubs.expect_container_spec)
1155         stubs.api.container_requests().create.assert_called_with(
1156             body=JsonDiffMatcher(expect_container))
1157         self.assertEqual(stubs.capture_stdout.getvalue(),
1158                          stubs.expect_container_request_uuid + '\n')
1159         self.assertEqual(exited, 0)
1160
1161     @stubs
1162     def test_submit_wf_runner_resources(self, stubs):
1163         exited = arvados_cwl.main(
1164             ["--submit", "--no-wait", "--api=containers", "--debug",
1165                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1166             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1167
1168         expect_container = copy.deepcopy(stubs.expect_container_spec)
1169         expect_container["runtime_constraints"] = {
1170             "API": True,
1171             "vcpus": 2,
1172             "ram": (2000+512) * 2**20
1173         }
1174         expect_container["name"] = "submit_wf_runner_resources.cwl"
1175         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1176             {
1177                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1178                 "coresMin": 2,
1179                 "ramMin": 2000,
1180                 "keep_cache": 512
1181             }
1182         ]
1183         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
1184             "arv": "http://arvados.org/cwl#",
1185         }
1186         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1187                         '--no-log-timestamps', '--disable-validate',
1188                         '--eval-timeout=20', '--thread-count=1',
1189                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1190                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1191
1192         stubs.api.container_requests().create.assert_called_with(
1193             body=JsonDiffMatcher(expect_container))
1194         self.assertEqual(stubs.capture_stdout.getvalue(),
1195                          stubs.expect_container_request_uuid + '\n')
1196         self.assertEqual(exited, 0)
1197
1198     def tearDown(self):
1199         arvados_cwl.arvdocker.arv_docker_clear_cache()
1200
1201     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1202     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1203     @mock.patch("arvados.api")
1204     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1205         arvados_cwl.arvdocker.arv_docker_clear_cache()
1206
1207         arvrunner = mock.MagicMock()
1208         arvrunner.project_uuid = ""
1209         api.return_value = mock.MagicMock()
1210         arvrunner.api = api.return_value
1211         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1212                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1213                                                                         "link_class": "docker_image_repo+tag",
1214                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1215                                                                         "owner_uuid": "",
1216                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1217                                                             {"items": [{"created_at": "",
1218                                                                         "head_uuid": "",
1219                                                                         "link_class": "docker_image_hash",
1220                                                                         "name": "123456",
1221                                                                         "owner_uuid": "",
1222                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1223         )
1224         find_one_image_hash.return_value = "123456"
1225
1226         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1227                                                                               "owner_uuid": "",
1228                                                                               "manifest_text": "",
1229                                                                               "properties": ""
1230                                                                           }], "items_available": 1, "offset": 0},)
1231         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1232         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1233                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1234         self.assertEqual("9999999999999999999999999999999b+99",
1235                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1236
1237
1238     @stubs
1239     def test_submit_secrets(self, stubs):
1240         exited = arvados_cwl.main(
1241             ["--submit", "--no-wait", "--api=containers", "--debug",
1242                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1243             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1244
1245         expect_container = {
1246             "command": [
1247                 "arvados-cwl-runner",
1248                 "--local",
1249                 "--api=containers",
1250                 "--no-log-timestamps",
1251                 "--disable-validate",
1252                 "--eval-timeout=20",
1253                 '--thread-count=1',
1254                 "--enable-reuse",
1255                 "--collection-cache-size=256",
1256                 '--debug',
1257                 "--on-error=continue",
1258                 "/var/lib/cwl/workflow.json#main",
1259                 "/var/lib/cwl/cwl.input.json"
1260             ],
1261             "container_image": "999999999999999999999999999999d3+99",
1262             "cwd": "/var/spool/cwl",
1263             "mounts": {
1264                 "/var/lib/cwl/cwl.input.json": {
1265                     "content": {
1266                         "pw": {
1267                             "$include": "/secrets/s0"
1268                         }
1269                     },
1270                     "kind": "json"
1271                 },
1272                 "/var/lib/cwl/workflow.json": {
1273                     "content": {
1274                         "$graph": [
1275                             {
1276                                 "$namespaces": {
1277                                     "cwltool": "http://commonwl.org/cwltool#"
1278                                 },
1279                                 "arguments": [
1280                                     "md5sum",
1281                                     "example.conf"
1282                                 ],
1283                                 "class": "CommandLineTool",
1284                                 "hints": [
1285                                     {
1286                                         "class": "http://commonwl.org/cwltool#Secrets",
1287                                         "secrets": [
1288                                             "#secret_job.cwl/pw"
1289                                         ]
1290                                     }
1291                                 ],
1292                                 "id": "#secret_job.cwl",
1293                                 "inputs": [
1294                                     {
1295                                         "id": "#secret_job.cwl/pw",
1296                                         "type": "string"
1297                                     }
1298                                 ],
1299                                 "outputs": [
1300                                     {
1301                                         "id": "#secret_job.cwl/out",
1302                                         "type": "stdout"
1303                                     }
1304                                 ],
1305                                 "stdout": "hashed_example.txt",
1306                                 "requirements": [
1307                                     {
1308                                         "class": "InitialWorkDirRequirement",
1309                                         "listing": [
1310                                             {
1311                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1312                                                 "entryname": "example.conf"
1313                                             }
1314                                         ]
1315                                     }
1316                                 ]
1317                             },
1318                             {
1319                                 "class": "Workflow",
1320                                 "hints": [
1321                                     {
1322                                         "class": "DockerRequirement",
1323                                         "dockerPull": "debian:8",
1324                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1325                                     },
1326                                     {
1327                                         "class": "http://commonwl.org/cwltool#Secrets",
1328                                         "secrets": [
1329                                             "#main/pw"
1330                                         ]
1331                                     }
1332                                 ],
1333                                 "id": "#main",
1334                                 "inputs": [
1335                                     {
1336                                         "id": "#main/pw",
1337                                         "type": "string"
1338                                     }
1339                                 ],
1340                                 "outputs": [
1341                                     {
1342                                         "id": "#main/out",
1343                                         "outputSource": "#main/step1/out",
1344                                         "type": "File"
1345                                     }
1346                                 ],
1347                                 "steps": [
1348                                     {
1349                                         "id": "#main/step1",
1350                                         "in": [
1351                                             {
1352                                                 "id": "#main/step1/pw",
1353                                                 "source": "#main/pw"
1354                                             }
1355                                         ],
1356                                         "out": [
1357                                             "#main/step1/out"
1358                                         ],
1359                                         "run": "#secret_job.cwl"
1360                                     }
1361                                 ]
1362                             }
1363                         ],
1364                         "cwlVersion": "v1.0"
1365                     },
1366                     "kind": "json"
1367                 },
1368                 "/var/spool/cwl": {
1369                     "kind": "collection",
1370                     "writable": True
1371                 },
1372                 "stdout": {
1373                     "kind": "file",
1374                     "path": "/var/spool/cwl/cwl.output.json"
1375                 }
1376             },
1377             "name": "secret_wf.cwl",
1378             "output_path": "/var/spool/cwl",
1379             "priority": 500,
1380             "properties": {},
1381             "runtime_constraints": {
1382                 "API": True,
1383                 "ram": 1342177280,
1384                 "vcpus": 1
1385             },
1386             "secret_mounts": {
1387                 "/secrets/s0": {
1388                     "content": "blorp",
1389                     "kind": "text"
1390                 }
1391             },
1392             "state": "Committed",
1393             "use_existing": True
1394         }
1395
1396         stubs.api.container_requests().create.assert_called_with(
1397             body=JsonDiffMatcher(expect_container))
1398         self.assertEqual(stubs.capture_stdout.getvalue(),
1399                          stubs.expect_container_request_uuid + '\n')
1400         self.assertEqual(exited, 0)
1401
1402     @stubs
1403     def test_submit_request_uuid(self, stubs):
1404         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1405         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1406
1407         stubs.api.container_requests().update().execute.return_value = {
1408             "uuid": stubs.expect_container_request_uuid,
1409             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1410             "state": "Queued"
1411         }
1412
1413         exited = arvados_cwl.main(
1414             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1415                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1416             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1417
1418         stubs.api.container_requests().update.assert_called_with(
1419             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1420         self.assertEqual(stubs.capture_stdout.getvalue(),
1421                          stubs.expect_container_request_uuid + '\n')
1422         self.assertEqual(exited, 0)
1423
1424     @stubs
1425     def test_submit_container_cluster_id(self, stubs):
1426         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1427
1428         exited = arvados_cwl.main(
1429             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1430                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1431             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1432
1433         expect_container = copy.deepcopy(stubs.expect_container_spec)
1434
1435         stubs.api.container_requests().create.assert_called_with(
1436             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1437         self.assertEqual(stubs.capture_stdout.getvalue(),
1438                          stubs.expect_container_request_uuid + '\n')
1439         self.assertEqual(exited, 0)
1440
1441     @stubs
1442     def test_submit_validate_cluster_id(self, stubs):
1443         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1444         exited = arvados_cwl.main(
1445             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1446              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1447             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1448         self.assertEqual(exited, 1)
1449
1450     @mock.patch("arvados.collection.CollectionReader")
1451     @stubs
1452     def test_submit_uuid_inputs(self, stubs, collectionReader):
1453         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1454         def list_side_effect(**kwargs):
1455             m = mock.MagicMock()
1456             if "count" in kwargs:
1457                 m.execute.return_value = {"items": [
1458                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1459                 ]}
1460             else:
1461                 m.execute.return_value = {"items": []}
1462             return m
1463         stubs.api.collections().list.side_effect = list_side_effect
1464
1465         exited = arvados_cwl.main(
1466             ["--submit", "--no-wait", "--api=containers", "--debug",
1467                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1468             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1469
1470         expect_container = copy.deepcopy(stubs.expect_container_spec)
1471         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1472         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1473         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1474
1475         stubs.api.collections().list.assert_has_calls([
1476             mock.call(count='none',
1477                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1478                       select=['uuid', 'portable_data_hash'])])
1479         stubs.api.container_requests().create.assert_called_with(
1480             body=JsonDiffMatcher(expect_container))
1481         self.assertEqual(stubs.capture_stdout.getvalue(),
1482                          stubs.expect_container_request_uuid + '\n')
1483         self.assertEqual(exited, 0)
1484
1485     @stubs
1486     def test_submit_mismatched_uuid_inputs(self, stubs):
1487         def list_side_effect(**kwargs):
1488             m = mock.MagicMock()
1489             if "count" in kwargs:
1490                 m.execute.return_value = {"items": [
1491                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1492                 ]}
1493             else:
1494                 m.execute.return_value = {"items": []}
1495             return m
1496         stubs.api.collections().list.side_effect = list_side_effect
1497
1498         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1499             capture_stderr = StringIO()
1500             cwltool_logger = logging.getLogger('cwltool')
1501             stderr_logger = logging.StreamHandler(capture_stderr)
1502             cwltool_logger.addHandler(stderr_logger)
1503
1504             try:
1505                 exited = arvados_cwl.main(
1506                     ["--submit", "--no-wait", "--api=containers", "--debug",
1507                         "tests/wf/submit_wf.cwl", infile],
1508                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1509
1510                 self.assertEqual(exited, 1)
1511                 self.assertRegexpMatches(
1512                     capture_stderr.getvalue(),
1513                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1514             finally:
1515                 cwltool_logger.removeHandler(stderr_logger)
1516
1517     @mock.patch("arvados.collection.CollectionReader")
1518     @stubs
1519     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1520         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1521         capture_stderr = StringIO()
1522
1523         cwltool_logger = logging.getLogger('cwltool')
1524         stderr_logger = logging.StreamHandler(capture_stderr)
1525         cwltool_logger.addHandler(stderr_logger)
1526
1527         exited = arvados_cwl.main(
1528             ["--submit", "--no-wait", "--api=containers", "--debug",
1529                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1530             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1531
1532         try:
1533             self.assertEqual(exited, 1)
1534             self.assertRegexpMatches(
1535                 capture_stderr.getvalue(),
1536                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1537         finally:
1538             cwltool_logger.removeHandler(stderr_logger)
1539
1540
1541 class TestCreateTemplate(unittest.TestCase):
1542     existing_template_uuid = "zzzzz-p5p6p-validworkfloyml"
1543
1544     def _adjust_script_params(self, expect_component):
1545         expect_component['script_parameters']['x'] = {
1546             'dataclass': 'File',
1547             'required': True,
1548             'type': 'File',
1549             'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
1550         }
1551         expect_component['script_parameters']['y'] = {
1552             'dataclass': 'Collection',
1553             'required': True,
1554             'type': 'Directory',
1555             'value': '99999999999999999999999999999998+99',
1556         }
1557         expect_component['script_parameters']['z'] = {
1558             'dataclass': 'Collection',
1559             'required': True,
1560             'type': 'Directory',
1561         }
1562
1563     @stubs
1564     def test_create(self, stubs):
1565         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1566
1567         exited = arvados_cwl.main(
1568             ["--create-workflow", "--debug",
1569              "--api=jobs",
1570              "--project-uuid", project_uuid,
1571              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1572             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1573
1574         stubs.api.pipeline_instances().create.refute_called()
1575         stubs.api.jobs().create.refute_called()
1576
1577         expect_component = copy.deepcopy(stubs.expect_job_spec)
1578         self._adjust_script_params(expect_component)
1579         expect_template = {
1580             "components": {
1581                 "submit_wf.cwl": expect_component,
1582             },
1583             "name": "submit_wf.cwl",
1584             "owner_uuid": project_uuid,
1585         }
1586         stubs.api.pipeline_templates().create.assert_called_with(
1587             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1588
1589         self.assertEqual(stubs.capture_stdout.getvalue(),
1590                          stubs.expect_pipeline_template_uuid + '\n')
1591         self.assertEqual(exited, 0)
1592
1593     @stubs
1594     def test_create_name(self, stubs):
1595         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1596
1597         exited = arvados_cwl.main(
1598             ["--create-workflow", "--debug",
1599              "--project-uuid", project_uuid,
1600              "--api=jobs",
1601              "--name", "testing 123",
1602              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1603             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1604
1605         stubs.api.pipeline_instances().create.refute_called()
1606         stubs.api.jobs().create.refute_called()
1607
1608         expect_component = copy.deepcopy(stubs.expect_job_spec)
1609         self._adjust_script_params(expect_component)
1610         expect_template = {
1611             "components": {
1612                 "testing 123": expect_component,
1613             },
1614             "name": "testing 123",
1615             "owner_uuid": project_uuid,
1616         }
1617         stubs.api.pipeline_templates().create.assert_called_with(
1618             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1619
1620         self.assertEqual(stubs.capture_stdout.getvalue(),
1621                          stubs.expect_pipeline_template_uuid + '\n')
1622         self.assertEqual(exited, 0)
1623
1624     @stubs
1625     def test_update_name(self, stubs):
1626         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1627
1628         exited = arvados_cwl.main(
1629             ["--update-workflow", self.existing_template_uuid,
1630              "--debug",
1631              "--project-uuid", project_uuid,
1632              "--api=jobs",
1633              "--name", "testing 123",
1634              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1635             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1636
1637         stubs.api.pipeline_instances().create.refute_called()
1638         stubs.api.jobs().create.refute_called()
1639
1640         expect_component = copy.deepcopy(stubs.expect_job_spec)
1641         self._adjust_script_params(expect_component)
1642         expect_template = {
1643             "components": {
1644                 "testing 123": expect_component,
1645             },
1646             "name": "testing 123",
1647             "owner_uuid": project_uuid,
1648         }
1649         stubs.api.pipeline_templates().create.refute_called()
1650         stubs.api.pipeline_templates().update.assert_called_with(
1651             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
1652
1653         self.assertEqual(stubs.capture_stdout.getvalue(),
1654                          self.existing_template_uuid + '\n')
1655         self.assertEqual(exited, 0)
1656
1657
1658 class TestCreateWorkflow(unittest.TestCase):
1659     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1660     expect_workflow = StripYAMLComments(
1661         open("tests/wf/expect_packed.cwl").read())
1662
1663     @stubs
1664     def test_create(self, stubs):
1665         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1666
1667         exited = arvados_cwl.main(
1668             ["--create-workflow", "--debug",
1669              "--api=containers",
1670              "--project-uuid", project_uuid,
1671              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1672             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1673
1674         stubs.api.pipeline_templates().create.refute_called()
1675         stubs.api.container_requests().create.refute_called()
1676
1677         body = {
1678             "workflow": {
1679                 "owner_uuid": project_uuid,
1680                 "name": "submit_wf.cwl",
1681                 "description": "",
1682                 "definition": self.expect_workflow,
1683             }
1684         }
1685         stubs.api.workflows().create.assert_called_with(
1686             body=JsonDiffMatcher(body))
1687
1688         self.assertEqual(stubs.capture_stdout.getvalue(),
1689                          stubs.expect_workflow_uuid + '\n')
1690         self.assertEqual(exited, 0)
1691
1692     @stubs
1693     def test_create_name(self, stubs):
1694         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1695
1696         exited = arvados_cwl.main(
1697             ["--create-workflow", "--debug",
1698              "--api=containers",
1699              "--project-uuid", project_uuid,
1700              "--name", "testing 123",
1701              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1702             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1703
1704         stubs.api.pipeline_templates().create.refute_called()
1705         stubs.api.container_requests().create.refute_called()
1706
1707         body = {
1708             "workflow": {
1709                 "owner_uuid": project_uuid,
1710                 "name": "testing 123",
1711                 "description": "",
1712                 "definition": self.expect_workflow,
1713             }
1714         }
1715         stubs.api.workflows().create.assert_called_with(
1716             body=JsonDiffMatcher(body))
1717
1718         self.assertEqual(stubs.capture_stdout.getvalue(),
1719                          stubs.expect_workflow_uuid + '\n')
1720         self.assertEqual(exited, 0)
1721
1722     @stubs
1723     def test_incompatible_api(self, stubs):
1724         capture_stderr = StringIO()
1725         acr_logger = logging.getLogger('arvados.cwl-runner')
1726         stderr_logger = logging.StreamHandler(capture_stderr)
1727         acr_logger.addHandler(stderr_logger)
1728
1729         try:
1730             exited = arvados_cwl.main(
1731                 ["--update-workflow", self.existing_workflow_uuid,
1732                  "--api=jobs",
1733                  "--debug",
1734                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1735                 sys.stderr, sys.stderr, api_client=stubs.api)
1736             self.assertEqual(exited, 1)
1737             self.assertRegexpMatches(
1738                 capture_stderr.getvalue(),
1739                 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1740         finally:
1741             acr_logger.removeHandler(stderr_logger)
1742
1743     @stubs
1744     def test_update(self, stubs):
1745         exited = arvados_cwl.main(
1746             ["--update-workflow", self.existing_workflow_uuid,
1747              "--debug",
1748              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1749             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1750
1751         body = {
1752             "workflow": {
1753                 "name": "submit_wf.cwl",
1754                 "description": "",
1755                 "definition": self.expect_workflow,
1756             }
1757         }
1758         stubs.api.workflows().update.assert_called_with(
1759             uuid=self.existing_workflow_uuid,
1760             body=JsonDiffMatcher(body))
1761         self.assertEqual(stubs.capture_stdout.getvalue(),
1762                          self.existing_workflow_uuid + '\n')
1763         self.assertEqual(exited, 0)
1764
1765     @stubs
1766     def test_update_name(self, stubs):
1767         exited = arvados_cwl.main(
1768             ["--update-workflow", self.existing_workflow_uuid,
1769              "--debug", "--name", "testing 123",
1770              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1771             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1772
1773         body = {
1774             "workflow": {
1775                 "name": "testing 123",
1776                 "description": "",
1777                 "definition": self.expect_workflow,
1778             }
1779         }
1780         stubs.api.workflows().update.assert_called_with(
1781             uuid=self.existing_workflow_uuid,
1782             body=JsonDiffMatcher(body))
1783         self.assertEqual(stubs.capture_stdout.getvalue(),
1784                          self.existing_workflow_uuid + '\n')
1785         self.assertEqual(exited, 0)
1786
1787     @stubs
1788     def test_create_collection_per_tool(self, stubs):
1789         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1790
1791         exited = arvados_cwl.main(
1792             ["--create-workflow", "--debug",
1793              "--api=containers",
1794              "--project-uuid", project_uuid,
1795              "tests/collection_per_tool/collection_per_tool.cwl"],
1796             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1797
1798         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1799         expect_workflow = StripYAMLComments(open(toolfile).read())
1800
1801         body = {
1802             "workflow": {
1803                 "owner_uuid": project_uuid,
1804                 "name": "collection_per_tool.cwl",
1805                 "description": "",
1806                 "definition": expect_workflow,
1807             }
1808         }
1809         stubs.api.workflows().create.assert_called_with(
1810             body=JsonDiffMatcher(body))
1811
1812         self.assertEqual(stubs.capture_stdout.getvalue(),
1813                          stubs.expect_workflow_uuid + '\n')
1814         self.assertEqual(exited, 0)
1815
1816 class TestTemplateInputs(unittest.TestCase):
1817     expect_template = {
1818         "components": {
1819             "inputs_test.cwl": {
1820                 'runtime_constraints': {
1821                     'docker_image': '999999999999999999999999999999d3+99',
1822                     'min_ram_mb_per_node': 1024
1823                 },
1824                 'script_parameters': {
1825                     'cwl:tool':
1826                     'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
1827                     'optionalFloatInput': None,
1828                     'fileInput': {
1829                         'type': 'File',
1830                         'dataclass': 'File',
1831                         'required': True,
1832                         'title': "It's a file; we expect to find some characters in it.",
1833                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1834                     },
1835                     'floatInput': {
1836                         'type': 'float',
1837                         'dataclass': 'number',
1838                         'required': True,
1839                         'title': 'Floats like a duck',
1840                         'default': 0.1,
1841                         'value': 0.1,
1842                     },
1843                     'optionalFloatInput': {
1844                         'type': ['null', 'float'],
1845                         'dataclass': 'number',
1846                         'required': False,
1847                     },
1848                     'boolInput': {
1849                         'type': 'boolean',
1850                         'dataclass': 'boolean',
1851                         'required': True,
1852                         'title': 'True or false?',
1853                     },
1854                 },
1855                 'repository': 'arvados',
1856                 'script_version': 'master',
1857                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1858                 'script': 'cwl-runner',
1859             },
1860         },
1861         "name": "inputs_test.cwl",
1862     }
1863
1864     @stubs
1865     def test_inputs_empty(self, stubs):
1866         exited = arvados_cwl.main(
1867             ["--debug", "--api=jobs", "--create-template",
1868              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1869             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1870
1871         stubs.api.pipeline_templates().create.assert_called_with(
1872             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1873
1874         self.assertEqual(exited, 0)
1875
1876     @stubs
1877     def test_inputs(self, stubs):
1878         exited = arvados_cwl.main(
1879             ["--api=jobs", "--create-template",
1880              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1881             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1882
1883         expect_template = copy.deepcopy(self.expect_template)
1884         params = expect_template[
1885             "components"]["inputs_test.cwl"]["script_parameters"]
1886         params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1887         params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
1888         params["floatInput"]["value"] = 1.234
1889         params["boolInput"]["value"] = True
1890
1891         stubs.api.pipeline_templates().create.assert_called_with(
1892             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
1893         self.assertEqual(exited, 0)