sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  51     @mock.patch("arvados.collection.KeepClient")
  52     @mock.patch("arvados.keep.KeepClient")
  53     @mock.patch("arvados.events.subscribe")
  54     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  55         class Stubs(object):
  56             pass
  57         stubs = Stubs()
  58         stubs.events = events
  59         stubs.keepdocker = keepdocker
  60
  61         def putstub(p, **kwargs):
  62             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  63         keep_client1().put.side_effect = putstub
  64         keep_client1.put.side_effect = putstub
  65         keep_client2().put.side_effect = putstub
  66         keep_client2.put.side_effect = putstub
  67
  68         stubs.keep_client = keep_client2
  69         stubs.docker_images = {
  70             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
  71             "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
  72             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
  73             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
  74         }
  75         def kd(a, b, image_name=None, image_tag=None):
  76             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  77         stubs.keepdocker.side_effect = kd
  78
  79         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  80         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  81
  82         if sys.version_info[0] < 3:
  83             stubs.capture_stdout = BytesIO()
  84         else:
  85             stubs.capture_stdout = StringIO()
  86
  87         stubs.api = mock.MagicMock()
  88         stubs.api._rootDesc = get_rootDesc()
  89         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  90
  91         stubs.api.users().current().execute.return_value = {
  92             "uuid": stubs.fake_user_uuid,
  93         }
  94         stubs.api.collections().list().execute.return_value = {"items": []}
  95         stubs.api.containers().current().execute.return_value = {
  96             "uuid": stubs.fake_container_uuid,
  97         }
  98
  99         class CollectionExecute(object):
 100             def __init__(self, exe):
 101                 self.exe = exe
 102             def execute(self, num_retries=None):
 103                 return self.exe
 104
 105         def collection_createstub(created_collections, body, ensure_unique_name=None):
 106             mt = body["manifest_text"].encode('utf-8')
 107             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 108             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 109             created_collections[uuid] = {
 110                 "uuid": uuid,
 111                 "portable_data_hash": pdh,
 112                 "manifest_text": mt.decode('utf-8')
 113             }
 114             return CollectionExecute(created_collections[uuid])
 115
 116         def collection_getstub(created_collections, uuid):
 117             for v in viewvalues(created_collections):
 118                 if uuid in (v["uuid"], v["portable_data_hash"]):
 119                     return CollectionExecute(v)
 120
 121         created_collections = {
 122             "99999999999999999999999999999998+99": {
 123                 "uuid": "",
 124                 "portable_data_hash": "99999999999999999999999999999998+99",
 125                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 126             },
 127             "99999999999999999999999999999997+99": {
 128                 "uuid": "",
 129                 "portable_data_hash": "99999999999999999999999999999997+99",
 130                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 131             },
 132             "99999999999999999999999999999994+99": {
 133                 "uuid": "",
 134                 "portable_data_hash": "99999999999999999999999999999994+99",
 135                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 136             },
 137             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 138                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 139                 "portable_data_hash": "999999999999999999999999999999d3+99",
 140                 "manifest_text": ""
 141             },
 142             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 143                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 144                 "portable_data_hash": "999999999999999999999999999999d4+99",
 145                 "manifest_text": ""
 146             },
 147             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 148                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 149                 "portable_data_hash": "999999999999999999999999999999d5+99",
 150                 "manifest_text": ""
 151             },
 152             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 153                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 154                 "portable_data_hash": "999999999999999999999999999999d6+99",
 155                 "manifest_text": ""
 156             }
 157         }
 158         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 159         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 160
 161         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         stubs.api.jobs().create().execute.return_value = {
 163             "uuid": stubs.expect_job_uuid,
 164             "state": "Queued",
 165         }
 166
 167         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 168         stubs.api.container_requests().create().execute.return_value = {
 169             "uuid": stubs.expect_container_request_uuid,
 170             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 171             "state": "Queued"
 172         }
 173
 174         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 175         stubs.api.pipeline_templates().create().execute.return_value = {
 176             "uuid": stubs.expect_pipeline_template_uuid,
 177         }
 178         stubs.expect_job_spec = {
 179             'runtime_constraints': {
 180                 'docker_image': '999999999999999999999999999999d3+99',
 181                 'min_ram_mb_per_node': 1024
 182             },
 183             'script_parameters': {
 184                 'x': {
 185                     'basename': 'blorp.txt',
 186                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 187                     'class': 'File'
 188                 },
 189                 'y': {
 190                     'basename': '99999999999999999999999999999998+99',
 191                     'location': 'keep:99999999999999999999999999999998+99',
 192                     'class': 'Directory'
 193                 },
 194                 'z': {
 195                     'basename': 'anonymous',
 196                     "listing": [{
 197                         "basename": "renamed.txt",
 198                         "class": "File",
 199                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 200                         "size": 0
 201                     }],
 202                     'class': 'Directory'
 203                 },
 204                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 205             },
 206             'repository': 'arvados',
 207             'script_version': 'master',
 208             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 209             'script': 'cwl-runner'
 210         }
 211         stubs.pipeline_component = stubs.expect_job_spec.copy()
 212         stubs.expect_pipeline_instance = {
 213             'name': 'submit_wf.cwl',
 214             'state': 'RunningOnServer',
 215             'owner_uuid': None,
 216             "components": {
 217                 "cwl-runner": {
 218                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 219                     'script_parameters': {
 220                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 221                         'x': {"value": {
 222                             'basename': 'blorp.txt',
 223                             'class': 'File',
 224                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 225                             "size": 16
 226                         }},
 227                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 228                               'listing': [
 229                                   {
 230                                       'basename': 'renamed.txt',
 231                                       'class': 'File', 'location':
 232                                       'keep:99999999999999999999999999999998+99/file1.txt',
 233                                       'size': 0
 234                                   }
 235                               ]}},
 236                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 237                         'arv:debug': True,
 238                         'arv:enable_reuse': True,
 239                         'arv:on_error': 'continue'
 240                     },
 241                     'repository': 'arvados',
 242                     'script_version': 'master',
 243                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 244                     'script': 'cwl-runner',
 245                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 246                 }
 247             }
 248         }
 249         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 250         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 251         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 252         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 253         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 254             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 255             "state": "Queued"
 256         }
 257         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 258         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 259
 260         with open("tests/wf/submit_wf_packed.cwl") as f:
 261             expect_packed_workflow = yaml.round_trip_load(f)
 262
 263         stubs.expect_container_spec = {
 264             'priority': 500,
 265             'mounts': {
 266                 '/var/spool/cwl': {
 267                     'writable': True,
 268                     'kind': 'collection'
 269                 },
 270                 '/var/lib/cwl/workflow.json': {
 271                     'content': expect_packed_workflow,
 272                     'kind': 'json'
 273                 },
 274                 'stdout': {
 275                     'path': '/var/spool/cwl/cwl.output.json',
 276                     'kind': 'file'
 277                 },
 278                 '/var/lib/cwl/cwl.input.json': {
 279                     'kind': 'json',
 280                     'content': {
 281                         'y': {
 282                             'basename': '99999999999999999999999999999998+99',
 283                             'location': 'keep:99999999999999999999999999999998+99',
 284                             'class': 'Directory'},
 285                         'x': {
 286                             'basename': u'blorp.txt',
 287                             'class': 'File',
 288                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 289                             "size": 16
 290                         },
 291                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 292                             {'basename': 'renamed.txt',
 293                              'class': 'File',
 294                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 295                              'size': 0
 296                             }
 297                         ]}
 298                     },
 299                     'kind': 'json'
 300                 }
 301             },
 302             'secret_mounts': {},
 303             'state': 'Committed',
 304             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 305                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 306                         '--eval-timeout=20', '--thread-count=1',
 307                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 308                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 309             'name': 'submit_wf.cwl',
 310             'container_image': '999999999999999999999999999999d3+99',
 311             'output_path': '/var/spool/cwl',
 312             'cwd': '/var/spool/cwl',
 313             'runtime_constraints': {
 314                 'API': True,
 315                 'vcpus': 1,
 316                 'ram': (1024+256)*1024*1024
 317             },
 318             'use_existing': False,
 319             'properties': {},
 320             'secret_mounts': {}
 321         }
 322
 323         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 324         stubs.api.workflows().create().execute.return_value = {
 325             "uuid": stubs.expect_workflow_uuid,
 326         }
 327         def update_mock(**kwargs):
 328             stubs.updated_uuid = kwargs.get('uuid')
 329             return mock.DEFAULT
 330         stubs.api.workflows().update.side_effect = update_mock
 331         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 332             "uuid": stubs.updated_uuid,
 333         }
 334
 335         return func(self, stubs, *args, **kwargs)
 336     return wrapped
 337
 338
 339 class TestSubmit(unittest.TestCase):
 340
 341     def setUp(self):
 342         cwltool.process._names = set()
 343
 344     @stubs
 345     def test_error_when_multiple_storage_classes_specified(self, stubs):
 346         storage_classes = "foo,bar"
 347         exited = arvados_cwl.main(
 348                 ["--debug", "--storage-classes", storage_classes,
 349                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 350                 sys.stdin, sys.stderr, api_client=stubs.api)
 351         self.assertEqual(exited, 1)
 352
 353     @mock.patch("time.sleep")
 354     @stubs
 355     def test_submit_invalid_runner_ram(self, stubs, tm):
 356         exited = arvados_cwl.main(
 357             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 358              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 359             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 360         self.assertEqual(exited, 1)
 361
 362
 363     @stubs
 364     def test_submit_container(self, stubs):
 365         exited = arvados_cwl.main(
 366             ["--submit", "--no-wait", "--api=containers", "--debug",
 367                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 368             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 369
 370         stubs.api.collections().create.assert_has_calls([
 371             mock.call(body=JsonDiffMatcher({
 372                 'manifest_text':
 373                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 374                 'replication_desired': None,
 375                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 376             }), ensure_unique_name=False),
 377             mock.call(body=JsonDiffMatcher({
 378                 'manifest_text':
 379                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 380                 'replication_desired': None,
 381                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 382             }), ensure_unique_name=False),
 383             ])
 384
 385         expect_container = copy.deepcopy(stubs.expect_container_spec)
 386         stubs.api.container_requests().create.assert_called_with(
 387             body=JsonDiffMatcher(expect_container))
 388         self.assertEqual(stubs.capture_stdout.getvalue(),
 389                          stubs.expect_container_request_uuid + '\n')
 390         self.assertEqual(exited, 0)
 391
 392
 393     @stubs
 394     def test_submit_container_tool(self, stubs):
 395         # test for issue #16139
 396         exited = arvados_cwl.main(
 397             ["--submit", "--no-wait", "--api=containers", "--debug",
 398                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 399             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 400
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403         self.assertEqual(exited, 0)
 404
 405     @stubs
 406     def test_submit_container_no_reuse(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = [
 414             'arvados-cwl-runner', '--local', '--api=containers',
 415             '--no-log-timestamps', '--disable-validate', '--disable-color',
 416             '--eval-timeout=20', '--thread-count=1',
 417             '--disable-reuse', "--collection-cache-size=256",
 418             '--debug', '--on-error=continue',
 419             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420         expect_container["use_existing"] = False
 421
 422         stubs.api.container_requests().create.assert_called_with(
 423             body=JsonDiffMatcher(expect_container))
 424         self.assertEqual(stubs.capture_stdout.getvalue(),
 425                          stubs.expect_container_request_uuid + '\n')
 426         self.assertEqual(exited, 0)
 427
 428     @stubs
 429     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 430         exited = arvados_cwl.main(
 431             ["--submit", "--no-wait", "--api=containers", "--debug",
 432              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 433             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 434         self.assertEqual(exited, 0)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = [
 438             'arvados-cwl-runner', '--local', '--api=containers',
 439             '--no-log-timestamps', '--disable-validate', '--disable-color',
 440             '--eval-timeout=20', '--thread-count=1',
 441             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 442             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["use_existing"] = False
 444         expect_container["name"] = "submit_wf_no_reuse.cwl"
 445         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 446             {
 447                 "class": "http://arvados.org/cwl#ReuseRequirement",
 448                 "enableReuse": False,
 449             },
 450         ]
 451         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 452             "arv": "http://arvados.org/cwl#",
 453             "cwltool": "http://commonwl.org/cwltool#"
 454         }
 455
 456         stubs.api.container_requests().create.assert_called_with(
 457             body=JsonDiffMatcher(expect_container))
 458         self.assertEqual(stubs.capture_stdout.getvalue(),
 459                          stubs.expect_container_request_uuid + '\n')
 460
 461
 462     @stubs
 463     def test_submit_container_on_error(self, stubs):
 464         exited = arvados_cwl.main(
 465             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 466                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468
 469         expect_container = copy.deepcopy(stubs.expect_container_spec)
 470         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 471                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 472                                        '--eval-timeout=20', '--thread-count=1',
 473                                        '--enable-reuse', "--collection-cache-size=256",
 474                                        '--debug', '--on-error=stop',
 475                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 476
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @stubs
 484     def test_submit_container_output_name(self, stubs):
 485         output_name = "test_output_name"
 486
 487         exited = arvados_cwl.main(
 488             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 489                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 490             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 491
 492         expect_container = copy.deepcopy(stubs.expect_container_spec)
 493         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 494                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 495                                        '--eval-timeout=20', '--thread-count=1',
 496                                        '--enable-reuse', "--collection-cache-size=256",
 497                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 498                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 499         expect_container["output_name"] = output_name
 500
 501         stubs.api.container_requests().create.assert_called_with(
 502             body=JsonDiffMatcher(expect_container))
 503         self.assertEqual(stubs.capture_stdout.getvalue(),
 504                          stubs.expect_container_request_uuid + '\n')
 505         self.assertEqual(exited, 0)
 506
 507     @stubs
 508     def test_submit_storage_classes(self, stubs):
 509         exited = arvados_cwl.main(
 510             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 511                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 512             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 513
 514         expect_container = copy.deepcopy(stubs.expect_container_spec)
 515         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 516                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 517                                        '--eval-timeout=20', '--thread-count=1',
 518                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 519                                        "--storage-classes=foo", '--on-error=continue',
 520                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 521
 522         stubs.api.container_requests().create.assert_called_with(
 523             body=JsonDiffMatcher(expect_container))
 524         self.assertEqual(stubs.capture_stdout.getvalue(),
 525                          stubs.expect_container_request_uuid + '\n')
 526         self.assertEqual(exited, 0)
 527
 528     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 529     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 530     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 531     @stubs
 532     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 533         final_output_c = arvados.collection.Collection()
 534         make_output.return_value = ({},final_output_c)
 535
 536         def set_final_output(job_order, output_callback, runtimeContext):
 537             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 538             return []
 539         job.side_effect = set_final_output
 540
 541         exited = arvados_cwl.main(
 542             ["--debug", "--local", "--storage-classes=foo",
 543                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 544             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 545
 546         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 547         self.assertEqual(exited, 0)
 548
 549     @mock.patch("arvados_cwl.task_queue.TaskQueue")
 550     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 551     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 552     @stubs
 553     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 554         final_output_c = arvados.collection.Collection()
 555         make_output.return_value = ({},final_output_c)
 556
 557         def set_final_output(job_order, output_callback, runtimeContext):
 558             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 559             return []
 560         job.side_effect = set_final_output
 561
 562         exited = arvados_cwl.main(
 563             ["--debug", "--local",
 564                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 565             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 566
 567         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 568         self.assertEqual(exited, 0)
 569
 570     @stubs
 571     def test_submit_container_output_ttl(self, stubs):
 572         exited = arvados_cwl.main(
 573             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         expect_container = copy.deepcopy(stubs.expect_container_spec)
 578         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 579                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 580                                        '--eval-timeout=20', '--thread-count=1',
 581                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 582                                        '--on-error=continue',
 583                                        "--intermediate-output-ttl=3600",
 584                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 585
 586         stubs.api.container_requests().create.assert_called_with(
 587             body=JsonDiffMatcher(expect_container))
 588         self.assertEqual(stubs.capture_stdout.getvalue(),
 589                          stubs.expect_container_request_uuid + '\n')
 590         self.assertEqual(exited, 0)
 591
 592     @stubs
 593     def test_submit_container_trash_intermediate(self, stubs):
 594         exited = arvados_cwl.main(
 595             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 596                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598
 599
 600         expect_container = copy.deepcopy(stubs.expect_container_spec)
 601         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 602                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 603                                        '--eval-timeout=20', '--thread-count=1',
 604                                        '--enable-reuse', "--collection-cache-size=256",
 605                                        '--debug', '--on-error=continue',
 606                                        "--trash-intermediate",
 607                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 608
 609         stubs.api.container_requests().create.assert_called_with(
 610             body=JsonDiffMatcher(expect_container))
 611         self.assertEqual(stubs.capture_stdout.getvalue(),
 612                          stubs.expect_container_request_uuid + '\n')
 613         self.assertEqual(exited, 0)
 614
 615     @stubs
 616     def test_submit_container_output_tags(self, stubs):
 617         output_tags = "tag0,tag1,tag2"
 618
 619         exited = arvados_cwl.main(
 620             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 621                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 622             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 623
 624         expect_container = copy.deepcopy(stubs.expect_container_spec)
 625         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 626                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 627                                        '--eval-timeout=20', '--thread-count=1',
 628                                        '--enable-reuse', "--collection-cache-size=256",
 629                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 630                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 631
 632         stubs.api.container_requests().create.assert_called_with(
 633             body=JsonDiffMatcher(expect_container))
 634         self.assertEqual(stubs.capture_stdout.getvalue(),
 635                          stubs.expect_container_request_uuid + '\n')
 636         self.assertEqual(exited, 0)
 637
 638     @stubs
 639     def test_submit_container_runner_ram(self, stubs):
 640         exited = arvados_cwl.main(
 641             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 642                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 643             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 644
 645         expect_container = copy.deepcopy(stubs.expect_container_spec)
 646         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 647
 648         stubs.api.container_requests().create.assert_called_with(
 649             body=JsonDiffMatcher(expect_container))
 650         self.assertEqual(stubs.capture_stdout.getvalue(),
 651                          stubs.expect_container_request_uuid + '\n')
 652         self.assertEqual(exited, 0)
 653
 654     @mock.patch("arvados.collection.CollectionReader")
 655     @mock.patch("time.sleep")
 656     @stubs
 657     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 658         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 659         exited = arvados_cwl.main(
 660             ["--submit", "--no-wait", "--api=containers", "--debug",
 661              "tests/wf/submit_keepref_wf.cwl"],
 662             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 663         self.assertEqual(exited, 0)
 664
 665     @mock.patch("arvados.collection.CollectionReader")
 666     @mock.patch("time.sleep")
 667     @stubs
 668     def test_submit_keepref(self, stubs, tm, reader):
 669         with open("tests/wf/expect_arvworkflow.cwl") as f:
 670             reader().open().__enter__().read.return_value = f.read()
 671
 672         exited = arvados_cwl.main(
 673             ["--submit", "--no-wait", "--api=containers", "--debug",
 674              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 676
 677         expect_container = {
 678             'priority': 500,
 679             'mounts': {
 680                 '/var/spool/cwl': {
 681                     'writable': True,
 682                     'kind': 'collection'
 683                 },
 684                 'stdout': {
 685                     'path': '/var/spool/cwl/cwl.output.json',
 686                     'kind': 'file'
 687                 },
 688                 '/var/lib/cwl/workflow': {
 689                     'portable_data_hash': '99999999999999999999999999999994+99',
 690                     'kind': 'collection'
 691                 },
 692                 '/var/lib/cwl/cwl.input.json': {
 693                     'content': {
 694                         'x': 'XxX'
 695                     },
 696                     'kind': 'json'
 697                 }
 698             }, 'state': 'Committed',
 699             'output_path': '/var/spool/cwl',
 700             'name': 'expect_arvworkflow.cwl#main',
 701             'container_image': '999999999999999999999999999999d3+99',
 702             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 703                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 704                         '--eval-timeout=20', '--thread-count=1',
 705                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 706                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 707             'cwd': '/var/spool/cwl',
 708             'runtime_constraints': {
 709                 'API': True,
 710                 'vcpus': 1,
 711                 'ram': 1342177280
 712             },
 713             'use_existing': False,
 714             'properties': {},
 715             'secret_mounts': {}
 716         }
 717
 718         stubs.api.container_requests().create.assert_called_with(
 719             body=JsonDiffMatcher(expect_container))
 720         self.assertEqual(stubs.capture_stdout.getvalue(),
 721                          stubs.expect_container_request_uuid + '\n')
 722         self.assertEqual(exited, 0)
 723
 724     @mock.patch("time.sleep")
 725     @stubs
 726     def test_submit_arvworkflow(self, stubs, tm):
 727         with open("tests/wf/expect_arvworkflow.cwl") as f:
 728             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 729
 730         exited = arvados_cwl.main(
 731             ["--submit", "--no-wait", "--api=containers", "--debug",
 732              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 733             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 734
 735         expect_container = {
 736             'priority': 500,
 737             'mounts': {
 738                 '/var/spool/cwl': {
 739                     'writable': True,
 740                     'kind': 'collection'
 741                 },
 742                 'stdout': {
 743                     'path': '/var/spool/cwl/cwl.output.json',
 744                     'kind': 'file'
 745                 },
 746                 '/var/lib/cwl/workflow.json': {
 747                     'kind': 'json',
 748                     'content': {
 749                         'cwlVersion': 'v1.0',
 750                         '$graph': [
 751                             {
 752                                 'id': '#main',
 753                                 'inputs': [
 754                                     {'type': 'string', 'id': '#main/x'}
 755                                 ],
 756                                 'steps': [
 757                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 758                                      'run': '#submit_tool.cwl',
 759                                      'id': '#main/step1',
 760                                      'out': []}
 761                                 ],
 762                                 'class': 'Workflow',
 763                                 'outputs': []
 764                             },
 765                             {
 766                                 'inputs': [
 767                                     {
 768                                         'inputBinding': {'position': 1},
 769                                         'type': 'string',
 770                                         'id': '#submit_tool.cwl/x'}
 771                                 ],
 772                                 'requirements': [
 773                                     {
 774                                         'dockerPull': 'debian:8',
 775                                         'class': 'DockerRequirement',
 776                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 777                                     }
 778                                 ],
 779                                 'id': '#submit_tool.cwl',
 780                                 'outputs': [],
 781                                 'baseCommand': 'cat',
 782                                 'class': 'CommandLineTool'
 783                             }
 784                         ]
 785                     }
 786                 },
 787                 '/var/lib/cwl/cwl.input.json': {
 788                     'content': {
 789                         'x': 'XxX'
 790                     },
 791                     'kind': 'json'
 792                 }
 793             }, 'state': 'Committed',
 794             'output_path': '/var/spool/cwl',
 795             'name': 'a test workflow',
 796             'container_image': "999999999999999999999999999999d3+99",
 797             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 798                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 799                         '--eval-timeout=20', '--thread-count=1',
 800                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 801                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 802             'cwd': '/var/spool/cwl',
 803             'runtime_constraints': {
 804                 'API': True,
 805                 'vcpus': 1,
 806                 'ram': 1342177280
 807             },
 808             'use_existing': False,
 809             'properties': {
 810                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 811             },
 812             'secret_mounts': {}
 813         }
 814
 815         stubs.api.container_requests().create.assert_called_with(
 816             body=JsonDiffMatcher(expect_container))
 817         self.assertEqual(stubs.capture_stdout.getvalue(),
 818                          stubs.expect_container_request_uuid + '\n')
 819         self.assertEqual(exited, 0)
 820
 821     @stubs
 822     def test_submit_container_name(self, stubs):
 823         exited = arvados_cwl.main(
 824             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 825                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 826             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 827
 828         expect_container = copy.deepcopy(stubs.expect_container_spec)
 829         expect_container["name"] = "hello container 123"
 830
 831         stubs.api.container_requests().create.assert_called_with(
 832             body=JsonDiffMatcher(expect_container))
 833         self.assertEqual(stubs.capture_stdout.getvalue(),
 834                          stubs.expect_container_request_uuid + '\n')
 835         self.assertEqual(exited, 0)
 836
 837     @stubs
 838     def test_submit_missing_input(self, stubs):
 839         exited = arvados_cwl.main(
 840             ["--submit", "--no-wait", "--api=containers", "--debug",
 841              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 842             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 843         self.assertEqual(exited, 0)
 844
 845         exited = arvados_cwl.main(
 846             ["--submit", "--no-wait", "--api=containers", "--debug",
 847              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 848             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 849         self.assertEqual(exited, 1)
 850
 851     @stubs
 852     def test_submit_container_project(self, stubs):
 853         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 854         exited = arvados_cwl.main(
 855             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 856                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 857             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 858
 859         expect_container = copy.deepcopy(stubs.expect_container_spec)
 860         expect_container["owner_uuid"] = project_uuid
 861         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 862                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 863                                        "--eval-timeout=20", "--thread-count=1",
 864                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 865                                        '--on-error=continue',
 866                                        '--project-uuid='+project_uuid,
 867                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 868
 869         stubs.api.container_requests().create.assert_called_with(
 870             body=JsonDiffMatcher(expect_container))
 871         self.assertEqual(stubs.capture_stdout.getvalue(),
 872                          stubs.expect_container_request_uuid + '\n')
 873         self.assertEqual(exited, 0)
 874
 875     @stubs
 876     def test_submit_container_eval_timeout(self, stubs):
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 879                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 884                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 885                                        '--eval-timeout=60.0', '--thread-count=1',
 886                                        '--enable-reuse', "--collection-cache-size=256",
 887                                        '--debug', '--on-error=continue',
 888                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 889
 890         stubs.api.container_requests().create.assert_called_with(
 891             body=JsonDiffMatcher(expect_container))
 892         self.assertEqual(stubs.capture_stdout.getvalue(),
 893                          stubs.expect_container_request_uuid + '\n')
 894         self.assertEqual(exited, 0)
 895
 896     @stubs
 897     def test_submit_container_collection_cache(self, stubs):
 898         exited = arvados_cwl.main(
 899             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 900                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 901             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 902
 903         expect_container = copy.deepcopy(stubs.expect_container_spec)
 904         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 905                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 906                                        '--eval-timeout=20', '--thread-count=1',
 907                                        '--enable-reuse', "--collection-cache-size=500",
 908                                        '--debug', '--on-error=continue',
 909                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 910         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 911
 912         stubs.api.container_requests().create.assert_called_with(
 913             body=JsonDiffMatcher(expect_container))
 914         self.assertEqual(stubs.capture_stdout.getvalue(),
 915                          stubs.expect_container_request_uuid + '\n')
 916         self.assertEqual(exited, 0)
 917
 918     @stubs
 919     def test_submit_container_thread_count(self, stubs):
 920         exited = arvados_cwl.main(
 921             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 922                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 923             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 924
 925         expect_container = copy.deepcopy(stubs.expect_container_spec)
 926         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 927                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 928                                        '--eval-timeout=20', '--thread-count=20',
 929                                        '--enable-reuse', "--collection-cache-size=256",
 930                                        '--debug', '--on-error=continue',
 931                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 932
 933         stubs.api.container_requests().create.assert_called_with(
 934             body=JsonDiffMatcher(expect_container))
 935         self.assertEqual(stubs.capture_stdout.getvalue(),
 936                          stubs.expect_container_request_uuid + '\n')
 937         self.assertEqual(exited, 0)
 938
 939     @stubs
 940     def test_submit_container_runner_image(self, stubs):
 941         exited = arvados_cwl.main(
 942             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 943                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 944             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 945
 946         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 947
 948         expect_container = copy.deepcopy(stubs.expect_container_spec)
 949         stubs.api.container_requests().create.assert_called_with(
 950             body=JsonDiffMatcher(expect_container))
 951         self.assertEqual(stubs.capture_stdout.getvalue(),
 952                          stubs.expect_container_request_uuid + '\n')
 953         self.assertEqual(exited, 0)
 954
 955     @stubs
 956     def test_submit_priority(self, stubs):
 957         exited = arvados_cwl.main(
 958             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 959                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 960             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 961
 962         stubs.expect_container_spec["priority"] = 669
 963
 964         expect_container = copy.deepcopy(stubs.expect_container_spec)
 965         stubs.api.container_requests().create.assert_called_with(
 966             body=JsonDiffMatcher(expect_container))
 967         self.assertEqual(stubs.capture_stdout.getvalue(),
 968                          stubs.expect_container_request_uuid + '\n')
 969         self.assertEqual(exited, 0)
 970
 971     @stubs
 972     def test_submit_wf_runner_resources(self, stubs):
 973         exited = arvados_cwl.main(
 974             ["--submit", "--no-wait", "--api=containers", "--debug",
 975                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 976             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 977
 978         expect_container = copy.deepcopy(stubs.expect_container_spec)
 979         expect_container["runtime_constraints"] = {
 980             "API": True,
 981             "vcpus": 2,
 982             "ram": (2000+512) * 2**20
 983         }
 984         expect_container["name"] = "submit_wf_runner_resources.cwl"
 985         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 986             {
 987                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 988                 "coresMin": 2,
 989                 "ramMin": 2000,
 990                 "keep_cache": 512
 991             }
 992         ]
 993         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
 994             "arv": "http://arvados.org/cwl#",
 995         }
 996         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
 997                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 998                         '--eval-timeout=20', '--thread-count=1',
 999                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1000                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1001
1002         stubs.api.container_requests().create.assert_called_with(
1003             body=JsonDiffMatcher(expect_container))
1004         self.assertEqual(stubs.capture_stdout.getvalue(),
1005                          stubs.expect_container_request_uuid + '\n')
1006         self.assertEqual(exited, 0)
1007
1008     def tearDown(self):
1009         arvados_cwl.arvdocker.arv_docker_clear_cache()
1010
1011     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1012     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1013     @mock.patch("arvados.api")
1014     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1015         arvados_cwl.arvdocker.arv_docker_clear_cache()
1016
1017         arvrunner = mock.MagicMock()
1018         arvrunner.project_uuid = ""
1019         api.return_value = mock.MagicMock()
1020         arvrunner.api = api.return_value
1021         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1022                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1023                                                                         "link_class": "docker_image_repo+tag",
1024                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1025                                                                         "owner_uuid": "",
1026                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1027                                                             {"items": [{"created_at": "",
1028                                                                         "head_uuid": "",
1029                                                                         "link_class": "docker_image_hash",
1030                                                                         "name": "123456",
1031                                                                         "owner_uuid": "",
1032                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1033         )
1034         find_one_image_hash.return_value = "123456"
1035
1036         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1037                                                                               "owner_uuid": "",
1038                                                                               "manifest_text": "",
1039                                                                               "properties": ""
1040                                                                           }], "items_available": 1, "offset": 0},)
1041         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1042         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1043                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1044         self.assertEqual("9999999999999999999999999999999b+99",
1045                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1046
1047
1048     @stubs
1049     def test_submit_secrets(self, stubs):
1050         exited = arvados_cwl.main(
1051             ["--submit", "--no-wait", "--api=containers", "--debug",
1052                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1053             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1054
1055         expect_container = {
1056             "command": [
1057                 "arvados-cwl-runner",
1058                 "--local",
1059                 "--api=containers",
1060                 "--no-log-timestamps",
1061                 "--disable-validate",
1062                 "--disable-color",
1063                 "--eval-timeout=20",
1064                 '--thread-count=1',
1065                 "--enable-reuse",
1066                 "--collection-cache-size=256",
1067                 '--debug',
1068                 "--on-error=continue",
1069                 "/var/lib/cwl/workflow.json#main",
1070                 "/var/lib/cwl/cwl.input.json"
1071             ],
1072             "container_image": "999999999999999999999999999999d3+99",
1073             "cwd": "/var/spool/cwl",
1074             "mounts": {
1075                 "/var/lib/cwl/cwl.input.json": {
1076                     "content": {
1077                         "pw": {
1078                             "$include": "/secrets/s0"
1079                         }
1080                     },
1081                     "kind": "json"
1082                 },
1083                 "/var/lib/cwl/workflow.json": {
1084                     "content": {
1085                         "$graph": [
1086                             {
1087                                 "$namespaces": {
1088                                     "cwltool": "http://commonwl.org/cwltool#"
1089                                 },
1090                                 "arguments": [
1091                                     "md5sum",
1092                                     "example.conf"
1093                                 ],
1094                                 "class": "CommandLineTool",
1095                                 "hints": [
1096                                     {
1097                                         "class": "http://commonwl.org/cwltool#Secrets",
1098                                         "secrets": [
1099                                             "#secret_job.cwl/pw"
1100                                         ]
1101                                     }
1102                                 ],
1103                                 "id": "#secret_job.cwl",
1104                                 "inputs": [
1105                                     {
1106                                         "id": "#secret_job.cwl/pw",
1107                                         "type": "string"
1108                                     }
1109                                 ],
1110                                 "outputs": [
1111                                     {
1112                                         "id": "#secret_job.cwl/out",
1113                                         "type": "File",
1114                                         "outputBinding": {
1115                                               "glob": "hashed_example.txt"
1116                                         }
1117                                     }
1118                                 ],
1119                                 "stdout": "hashed_example.txt",
1120                                 "requirements": [
1121                                     {
1122                                         "class": "InitialWorkDirRequirement",
1123                                         "listing": [
1124                                             {
1125                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1126                                                 "entryname": "example.conf"
1127                                             }
1128                                         ]
1129                                     }
1130                                 ]
1131                             },
1132                             {
1133                                 "class": "Workflow",
1134                                 "hints": [
1135                                     {
1136                                         "class": "DockerRequirement",
1137                                         "dockerPull": "debian:8",
1138                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1139                                     },
1140                                     {
1141                                         "class": "http://commonwl.org/cwltool#Secrets",
1142                                         "secrets": [
1143                                             "#main/pw"
1144                                         ]
1145                                     }
1146                                 ],
1147                                 "id": "#main",
1148                                 "inputs": [
1149                                     {
1150                                         "id": "#main/pw",
1151                                         "type": "string"
1152                                     }
1153                                 ],
1154                                 "outputs": [
1155                                     {
1156                                         "id": "#main/out",
1157                                         "outputSource": "#main/step1/out",
1158                                         "type": "File"
1159                                     }
1160                                 ],
1161                                 "steps": [
1162                                     {
1163                                         "id": "#main/step1",
1164                                         "in": [
1165                                             {
1166                                                 "id": "#main/step1/pw",
1167                                                 "source": "#main/pw"
1168                                             }
1169                                         ],
1170                                         "out": [
1171                                             "#main/step1/out"
1172                                         ],
1173                                         "run": "#secret_job.cwl"
1174                                     }
1175                                 ]
1176                             }
1177                         ],
1178                         "cwlVersion": "v1.0"
1179                     },
1180                     "kind": "json"
1181                 },
1182                 "/var/spool/cwl": {
1183                     "kind": "collection",
1184                     "writable": True
1185                 },
1186                 "stdout": {
1187                     "kind": "file",
1188                     "path": "/var/spool/cwl/cwl.output.json"
1189                 }
1190             },
1191             "name": "secret_wf.cwl",
1192             "output_path": "/var/spool/cwl",
1193             "priority": 500,
1194             "properties": {},
1195             "runtime_constraints": {
1196                 "API": True,
1197                 "ram": 1342177280,
1198                 "vcpus": 1
1199             },
1200             "secret_mounts": {
1201                 "/secrets/s0": {
1202                     "content": "blorp",
1203                     "kind": "text"
1204                 }
1205             },
1206             "state": "Committed",
1207             "use_existing": False
1208         }
1209
1210         stubs.api.container_requests().create.assert_called_with(
1211             body=JsonDiffMatcher(expect_container))
1212         self.assertEqual(stubs.capture_stdout.getvalue(),
1213                          stubs.expect_container_request_uuid + '\n')
1214         self.assertEqual(exited, 0)
1215
1216     @stubs
1217     def test_submit_request_uuid(self, stubs):
1218         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1219         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1220
1221         stubs.api.container_requests().update().execute.return_value = {
1222             "uuid": stubs.expect_container_request_uuid,
1223             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1224             "state": "Queued"
1225         }
1226
1227         exited = arvados_cwl.main(
1228             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1229                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1230             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1231
1232         stubs.api.container_requests().update.assert_called_with(
1233             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1234         self.assertEqual(stubs.capture_stdout.getvalue(),
1235                          stubs.expect_container_request_uuid + '\n')
1236         self.assertEqual(exited, 0)
1237
1238     @stubs
1239     def test_submit_container_cluster_id(self, stubs):
1240         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1241
1242         exited = arvados_cwl.main(
1243             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1244                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1245             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1246
1247         expect_container = copy.deepcopy(stubs.expect_container_spec)
1248
1249         stubs.api.container_requests().create.assert_called_with(
1250             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1251         self.assertEqual(stubs.capture_stdout.getvalue(),
1252                          stubs.expect_container_request_uuid + '\n')
1253         self.assertEqual(exited, 0)
1254
1255     @stubs
1256     def test_submit_validate_cluster_id(self, stubs):
1257         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1258         exited = arvados_cwl.main(
1259             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1260              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1261             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1262         self.assertEqual(exited, 1)
1263
1264     @mock.patch("arvados.collection.CollectionReader")
1265     @stubs
1266     def test_submit_uuid_inputs(self, stubs, collectionReader):
1267         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1268         def list_side_effect(**kwargs):
1269             m = mock.MagicMock()
1270             if "count" in kwargs:
1271                 m.execute.return_value = {"items": [
1272                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1273                 ]}
1274             else:
1275                 m.execute.return_value = {"items": []}
1276             return m
1277         stubs.api.collections().list.side_effect = list_side_effect
1278
1279         exited = arvados_cwl.main(
1280             ["--submit", "--no-wait", "--api=containers", "--debug",
1281                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1282             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1283
1284         expect_container = copy.deepcopy(stubs.expect_container_spec)
1285         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1286         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1287         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1288
1289         stubs.api.collections().list.assert_has_calls([
1290             mock.call(count='none',
1291                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1292                       select=['uuid', 'portable_data_hash'])])
1293         stubs.api.container_requests().create.assert_called_with(
1294             body=JsonDiffMatcher(expect_container))
1295         self.assertEqual(stubs.capture_stdout.getvalue(),
1296                          stubs.expect_container_request_uuid + '\n')
1297         self.assertEqual(exited, 0)
1298
1299     @stubs
1300     def test_submit_mismatched_uuid_inputs(self, stubs):
1301         def list_side_effect(**kwargs):
1302             m = mock.MagicMock()
1303             if "count" in kwargs:
1304                 m.execute.return_value = {"items": [
1305                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1306                 ]}
1307             else:
1308                 m.execute.return_value = {"items": []}
1309             return m
1310         stubs.api.collections().list.side_effect = list_side_effect
1311
1312         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1313             capture_stderr = StringIO()
1314             cwltool_logger = logging.getLogger('cwltool')
1315             stderr_logger = logging.StreamHandler(capture_stderr)
1316             cwltool_logger.addHandler(stderr_logger)
1317
1318             try:
1319                 exited = arvados_cwl.main(
1320                     ["--submit", "--no-wait", "--api=containers", "--debug",
1321                         "tests/wf/submit_wf.cwl", infile],
1322                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1323
1324                 self.assertEqual(exited, 1)
1325                 self.assertRegex(
1326                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1327                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1328             finally:
1329                 cwltool_logger.removeHandler(stderr_logger)
1330
1331     @mock.patch("arvados.collection.CollectionReader")
1332     @stubs
1333     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1334         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1335         capture_stderr = StringIO()
1336
1337         cwltool_logger = logging.getLogger('cwltool')
1338         stderr_logger = logging.StreamHandler(capture_stderr)
1339         cwltool_logger.addHandler(stderr_logger)
1340
1341         exited = arvados_cwl.main(
1342             ["--submit", "--no-wait", "--api=containers", "--debug",
1343                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1344             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1345
1346         try:
1347             self.assertEqual(exited, 1)
1348             self.assertRegex(
1349                 capture_stderr.getvalue(),
1350                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1351         finally:
1352             cwltool_logger.removeHandler(stderr_logger)
1353
1354
1355 class TestCreateWorkflow(unittest.TestCase):
1356     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1357     expect_workflow = StripYAMLComments(
1358         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1359
1360     @stubs
1361     def test_create(self, stubs):
1362         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1363
1364         exited = arvados_cwl.main(
1365             ["--create-workflow", "--debug",
1366              "--api=containers",
1367              "--project-uuid", project_uuid,
1368              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1369             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1370
1371         stubs.api.pipeline_templates().create.refute_called()
1372         stubs.api.container_requests().create.refute_called()
1373
1374         body = {
1375             "workflow": {
1376                 "owner_uuid": project_uuid,
1377                 "name": "submit_wf.cwl",
1378                 "description": "",
1379                 "definition": self.expect_workflow,
1380             }
1381         }
1382         stubs.api.workflows().create.assert_called_with(
1383             body=JsonDiffMatcher(body))
1384
1385         self.assertEqual(stubs.capture_stdout.getvalue(),
1386                          stubs.expect_workflow_uuid + '\n')
1387         self.assertEqual(exited, 0)
1388
1389     @stubs
1390     def test_create_name(self, stubs):
1391         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1392
1393         exited = arvados_cwl.main(
1394             ["--create-workflow", "--debug",
1395              "--api=containers",
1396              "--project-uuid", project_uuid,
1397              "--name", "testing 123",
1398              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1399             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1400
1401         stubs.api.pipeline_templates().create.refute_called()
1402         stubs.api.container_requests().create.refute_called()
1403
1404         body = {
1405             "workflow": {
1406                 "owner_uuid": project_uuid,
1407                 "name": "testing 123",
1408                 "description": "",
1409                 "definition": self.expect_workflow,
1410             }
1411         }
1412         stubs.api.workflows().create.assert_called_with(
1413             body=JsonDiffMatcher(body))
1414
1415         self.assertEqual(stubs.capture_stdout.getvalue(),
1416                          stubs.expect_workflow_uuid + '\n')
1417         self.assertEqual(exited, 0)
1418
1419
1420     @stubs
1421     def test_update(self, stubs):
1422         exited = arvados_cwl.main(
1423             ["--update-workflow", self.existing_workflow_uuid,
1424              "--debug",
1425              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1426             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1427
1428         body = {
1429             "workflow": {
1430                 "name": "submit_wf.cwl",
1431                 "description": "",
1432                 "definition": self.expect_workflow,
1433             }
1434         }
1435         stubs.api.workflows().update.assert_called_with(
1436             uuid=self.existing_workflow_uuid,
1437             body=JsonDiffMatcher(body))
1438         self.assertEqual(stubs.capture_stdout.getvalue(),
1439                          self.existing_workflow_uuid + '\n')
1440         self.assertEqual(exited, 0)
1441
1442     @stubs
1443     def test_update_name(self, stubs):
1444         exited = arvados_cwl.main(
1445             ["--update-workflow", self.existing_workflow_uuid,
1446              "--debug", "--name", "testing 123",
1447              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1448             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1449
1450         body = {
1451             "workflow": {
1452                 "name": "testing 123",
1453                 "description": "",
1454                 "definition": self.expect_workflow,
1455             }
1456         }
1457         stubs.api.workflows().update.assert_called_with(
1458             uuid=self.existing_workflow_uuid,
1459             body=JsonDiffMatcher(body))
1460         self.assertEqual(stubs.capture_stdout.getvalue(),
1461                          self.existing_workflow_uuid + '\n')
1462         self.assertEqual(exited, 0)
1463
1464     @stubs
1465     def test_create_collection_per_tool(self, stubs):
1466         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1467
1468         exited = arvados_cwl.main(
1469             ["--create-workflow", "--debug",
1470              "--api=containers",
1471              "--project-uuid", project_uuid,
1472              "tests/collection_per_tool/collection_per_tool.cwl"],
1473             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1474
1475         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1476         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1477
1478         body = {
1479             "workflow": {
1480                 "owner_uuid": project_uuid,
1481                 "name": "collection_per_tool.cwl",
1482                 "description": "",
1483                 "definition": expect_workflow,
1484             }
1485         }
1486         stubs.api.workflows().create.assert_called_with(
1487             body=JsonDiffMatcher(body))
1488
1489         self.assertEqual(stubs.capture_stdout.getvalue(),
1490                          stubs.expect_workflow_uuid + '\n')
1491         self.assertEqual(exited, 0)
1492
1493     @stubs
1494     def test_create_with_imports(self, stubs):
1495         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1496
1497         exited = arvados_cwl.main(
1498             ["--create-workflow", "--debug",
1499              "--api=containers",
1500              "--project-uuid", project_uuid,
1501              "tests/wf/feddemo/feddemo.cwl"],
1502             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1503
1504         stubs.api.pipeline_templates().create.refute_called()
1505         stubs.api.container_requests().create.refute_called()
1506
1507         self.assertEqual(stubs.capture_stdout.getvalue(),
1508                          stubs.expect_workflow_uuid + '\n')
1509         self.assertEqual(exited, 0)
1510
1511     @stubs
1512     def test_create_with_no_input(self, stubs):
1513         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1514
1515         exited = arvados_cwl.main(
1516             ["--create-workflow", "--debug",
1517              "--api=containers",
1518              "--project-uuid", project_uuid,
1519              "tests/wf/revsort/revsort.cwl"],
1520             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1521
1522         stubs.api.pipeline_templates().create.refute_called()
1523         stubs.api.container_requests().create.refute_called()
1524
1525         self.assertEqual(stubs.capture_stdout.getvalue(),
1526                          stubs.expect_workflow_uuid + '\n')
1527         self.assertEqual(exited, 0)