sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("uuid.uuid4")
  51     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  52     @mock.patch("arvados.collection.KeepClient")
  53     @mock.patch("arvados.keep.KeepClient")
  54     @mock.patch("arvados.events.subscribe")
  55     def wrapped(self, events, keep_client1, keep_client2, keepdocker, uuid4, *args, **kwargs):
  56         class Stubs(object):
  57             pass
  58         stubs = Stubs()
  59         stubs.events = events
  60         stubs.keepdocker = keepdocker
  61
  62         uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  63                              "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  64                              "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  65
  66         def putstub(p, **kwargs):
  67             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  68         keep_client1().put.side_effect = putstub
  69         keep_client1.put.side_effect = putstub
  70         keep_client2().put.side_effect = putstub
  71         keep_client2.put.side_effect = putstub
  72
  73         stubs.keep_client = keep_client2
  74         stubs.docker_images = {
  75             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  76             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  77             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  78             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  79         }
  80         def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  81             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  82         stubs.keepdocker.side_effect = kd
  83
  84         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  85         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  86
  87         if sys.version_info[0] < 3:
  88             stubs.capture_stdout = BytesIO()
  89         else:
  90             stubs.capture_stdout = StringIO()
  91
  92         stubs.api = mock.MagicMock()
  93         stubs.api._rootDesc = get_rootDesc()
  94         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  95         stubs.api._rootDesc["revision"] = "20210628"
  96
  97         stubs.api.users().current().execute.return_value = {
  98             "uuid": stubs.fake_user_uuid,
  99         }
 100         stubs.api.collections().list().execute.return_value = {"items": []}
 101         stubs.api.containers().current().execute.return_value = {
 102             "uuid": stubs.fake_container_uuid,
 103         }
 104         stubs.api.config()["StorageClasses"].items.return_value = {
 105             "default": {
 106                 "Default": True
 107             }
 108         }.items()
 109
 110         class CollectionExecute(object):
 111             def __init__(self, exe):
 112                 self.exe = exe
 113             def execute(self, num_retries=None):
 114                 return self.exe
 115
 116         def collection_createstub(created_collections, body, ensure_unique_name=None):
 117             mt = body["manifest_text"].encode('utf-8')
 118             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 119             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 120             created_collections[uuid] = {
 121                 "uuid": uuid,
 122                 "portable_data_hash": pdh,
 123                 "manifest_text": mt.decode('utf-8')
 124             }
 125             return CollectionExecute(created_collections[uuid])
 126
 127         def collection_getstub(created_collections, uuid):
 128             for v in viewvalues(created_collections):
 129                 if uuid in (v["uuid"], v["portable_data_hash"]):
 130                     return CollectionExecute(v)
 131
 132         created_collections = {
 133             "99999999999999999999999999999998+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999998+99",
 136                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999997+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999997+99",
 141                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 142             },
 143             "99999999999999999999999999999994+99": {
 144                 "uuid": "",
 145                 "portable_data_hash": "99999999999999999999999999999994+99",
 146                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 150                 "portable_data_hash": "999999999999999999999999999999d3+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 155                 "portable_data_hash": "999999999999999999999999999999d4+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 160                 "portable_data_hash": "999999999999999999999999999999d5+99",
 161                 "manifest_text": ""
 162             },
 163             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 164                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 165                 "portable_data_hash": "999999999999999999999999999999d6+99",
 166                 "manifest_text": ""
 167             }
 168         }
 169         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 170         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 171
 172         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 173         stubs.api.jobs().create().execute.return_value = {
 174             "uuid": stubs.expect_job_uuid,
 175             "state": "Queued",
 176         }
 177
 178         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 179         stubs.api.container_requests().create().execute.return_value = {
 180             "uuid": stubs.expect_container_request_uuid,
 181             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 182             "state": "Queued"
 183         }
 184
 185         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 186         stubs.api.pipeline_templates().create().execute.return_value = {
 187             "uuid": stubs.expect_pipeline_template_uuid,
 188         }
 189         stubs.expect_job_spec = {
 190             'runtime_constraints': {
 191                 'docker_image': '999999999999999999999999999999d3+99',
 192                 'min_ram_mb_per_node': 1024
 193             },
 194             'script_parameters': {
 195                 'x': {
 196                     'basename': 'blorp.txt',
 197                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 198                     'class': 'File'
 199                 },
 200                 'y': {
 201                     'basename': '99999999999999999999999999999998+99',
 202                     'location': 'keep:99999999999999999999999999999998+99',
 203                     'class': 'Directory'
 204                 },
 205                 'z': {
 206                     'basename': 'anonymous',
 207                     "listing": [{
 208                         "basename": "renamed.txt",
 209                         "class": "File",
 210                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 211                         "size": 0
 212                     }],
 213                     'class': 'Directory'
 214                 },
 215                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 216             },
 217             'repository': 'arvados',
 218             'script_version': 'master',
 219             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 220             'script': 'cwl-runner'
 221         }
 222         stubs.pipeline_component = stubs.expect_job_spec.copy()
 223         stubs.expect_pipeline_instance = {
 224             'name': 'submit_wf.cwl',
 225             'state': 'RunningOnServer',
 226             'owner_uuid': None,
 227             "components": {
 228                 "cwl-runner": {
 229                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 230                     'script_parameters': {
 231                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 232                         'x': {"value": {
 233                             'basename': 'blorp.txt',
 234                             'class': 'File',
 235                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 236                             "size": 16
 237                         }},
 238                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 239                               'listing': [
 240                                   {
 241                                       'basename': 'renamed.txt',
 242                                       'class': 'File', 'location':
 243                                       'keep:99999999999999999999999999999998+99/file1.txt',
 244                                       'size': 0
 245                                   }
 246                               ]}},
 247                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 248                         'arv:debug': True,
 249                         'arv:enable_reuse': True,
 250                         'arv:on_error': 'continue'
 251                     },
 252                     'repository': 'arvados',
 253                     'script_version': 'master',
 254                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 255                     'script': 'cwl-runner',
 256                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 257                 }
 258             }
 259         }
 260         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 261         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 262         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 263         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 264         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 265             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 266             "state": "Queued"
 267         }
 268         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 269         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 270
 271         with open("tests/wf/submit_wf_packed.cwl") as f:
 272             expect_packed_workflow = yaml.round_trip_load(f)
 273
 274         stubs.expect_container_spec = {
 275             'priority': 500,
 276             'mounts': {
 277                 '/var/spool/cwl': {
 278                     'writable': True,
 279                     'kind': 'collection'
 280                 },
 281                 '/var/lib/cwl/workflow.json': {
 282                     'content': expect_packed_workflow,
 283                     'kind': 'json'
 284                 },
 285                 'stdout': {
 286                     'path': '/var/spool/cwl/cwl.output.json',
 287                     'kind': 'file'
 288                 },
 289                 '/var/lib/cwl/cwl.input.json': {
 290                     'kind': 'json',
 291                     'content': {
 292                         'y': {
 293                             'basename': '99999999999999999999999999999998+99',
 294                             'location': 'keep:99999999999999999999999999999998+99',
 295                             'class': 'Directory'},
 296                         'x': {
 297                             'basename': u'blorp.txt',
 298                             'class': 'File',
 299                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 300                             "size": 16
 301                         },
 302                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 303                             {'basename': 'renamed.txt',
 304                              'class': 'File',
 305                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 306                              'size': 0
 307                             }
 308                         ]}
 309                     },
 310                     'kind': 'json'
 311                 }
 312             },
 313             'secret_mounts': {},
 314             'state': 'Committed',
 315             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 316                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 317                         '--eval-timeout=20', '--thread-count=0',
 318                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 319                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 320             'name': 'submit_wf.cwl',
 321             'container_image': '999999999999999999999999999999d3+99',
 322             'output_path': '/var/spool/cwl',
 323             'cwd': '/var/spool/cwl',
 324             'runtime_constraints': {
 325                 'API': True,
 326                 'vcpus': 1,
 327                 'ram': (1024+256)*1024*1024
 328             },
 329             'use_existing': False,
 330             'properties': {},
 331             'secret_mounts': {}
 332         }
 333
 334         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 335         stubs.api.workflows().create().execute.return_value = {
 336             "uuid": stubs.expect_workflow_uuid,
 337         }
 338         def update_mock(**kwargs):
 339             stubs.updated_uuid = kwargs.get('uuid')
 340             return mock.DEFAULT
 341         stubs.api.workflows().update.side_effect = update_mock
 342         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 343             "uuid": stubs.updated_uuid,
 344         }
 345
 346         return func(self, stubs, *args, **kwargs)
 347     return wrapped
 348
 349
 350 class TestSubmit(unittest.TestCase):
 351
 352     def setUp(self):
 353         cwltool.process._names = set()
 354         arvados_cwl.arvdocker.arv_docker_clear_cache()
 355
 356     def tearDown(self):
 357         root_logger = logging.getLogger('')
 358
 359         # Remove existing RuntimeStatusLoggingHandlers if they exist
 360         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 361         root_logger.handlers = handlers
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_invalid_runner_ram(self, stubs, tm):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 368              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 370         self.assertEqual(exited, 1)
 371
 372
 373     @stubs
 374     def test_submit_container(self, stubs):
 375         exited = arvados_cwl.main(
 376             ["--submit", "--no-wait", "--api=containers", "--debug",
 377                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 378             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 379
 380         stubs.api.collections().create.assert_has_calls([
 381             mock.call(body=JsonDiffMatcher({
 382                 'manifest_text':
 383                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 384                 'replication_desired': None,
 385                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 386             }), ensure_unique_name=False),
 387             mock.call(body=JsonDiffMatcher({
 388                 'manifest_text':
 389                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 390                 'replication_desired': None,
 391                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 392             }), ensure_unique_name=False),
 393             ])
 394
 395         expect_container = copy.deepcopy(stubs.expect_container_spec)
 396         stubs.api.container_requests().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_container))
 398         self.assertEqual(stubs.capture_stdout.getvalue(),
 399                          stubs.expect_container_request_uuid + '\n')
 400         self.assertEqual(exited, 0)
 401
 402
 403     @stubs
 404     def test_submit_container_tool(self, stubs):
 405         # test for issue #16139
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--api=containers", "--debug",
 408                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 410
 411         self.assertEqual(stubs.capture_stdout.getvalue(),
 412                          stubs.expect_container_request_uuid + '\n')
 413         self.assertEqual(exited, 0)
 414
 415     @stubs
 416     def test_submit_container_no_reuse(self, stubs):
 417         exited = arvados_cwl.main(
 418             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 419                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 420             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 421
 422         expect_container = copy.deepcopy(stubs.expect_container_spec)
 423         expect_container["command"] = [
 424             'arvados-cwl-runner', '--local', '--api=containers',
 425             '--no-log-timestamps', '--disable-validate', '--disable-color',
 426             '--eval-timeout=20', '--thread-count=0',
 427             '--disable-reuse', "--collection-cache-size=256",
 428             '--debug', '--on-error=continue',
 429             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 430         expect_container["use_existing"] = False
 431
 432         stubs.api.container_requests().create.assert_called_with(
 433             body=JsonDiffMatcher(expect_container))
 434         self.assertEqual(stubs.capture_stdout.getvalue(),
 435                          stubs.expect_container_request_uuid + '\n')
 436         self.assertEqual(exited, 0)
 437
 438     @stubs
 439     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 440         exited = arvados_cwl.main(
 441             ["--submit", "--no-wait", "--api=containers", "--debug",
 442              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 443             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 444         self.assertEqual(exited, 0)
 445
 446         expect_container = copy.deepcopy(stubs.expect_container_spec)
 447         expect_container["command"] = [
 448             'arvados-cwl-runner', '--local', '--api=containers',
 449             '--no-log-timestamps', '--disable-validate', '--disable-color',
 450             '--eval-timeout=20', '--thread-count=0',
 451             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 452             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 453         expect_container["use_existing"] = False
 454         expect_container["name"] = "submit_wf_no_reuse.cwl"
 455         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 456             {
 457                 "class": "http://arvados.org/cwl#ReuseRequirement",
 458                 "enableReuse": False,
 459             },
 460         ]
 461         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 462             "arv": "http://arvados.org/cwl#",
 463             "cwltool": "http://commonwl.org/cwltool#"
 464         }
 465
 466         stubs.api.container_requests().create.assert_called_with(
 467             body=JsonDiffMatcher(expect_container))
 468         self.assertEqual(stubs.capture_stdout.getvalue(),
 469                          stubs.expect_container_request_uuid + '\n')
 470
 471
 472     @stubs
 473     def test_submit_container_on_error(self, stubs):
 474         exited = arvados_cwl.main(
 475             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 476                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 477             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 478
 479         expect_container = copy.deepcopy(stubs.expect_container_spec)
 480         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 481                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 482                                        '--eval-timeout=20', '--thread-count=0',
 483                                        '--enable-reuse', "--collection-cache-size=256",
 484                                        '--debug', '--on-error=stop',
 485                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 486
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_container))
 489         self.assertEqual(stubs.capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491         self.assertEqual(exited, 0)
 492
 493     @stubs
 494     def test_submit_container_output_name(self, stubs):
 495         output_name = "test_output_name"
 496
 497         exited = arvados_cwl.main(
 498             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 499                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 500             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 501
 502         expect_container = copy.deepcopy(stubs.expect_container_spec)
 503         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 504                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 505                                        '--eval-timeout=20', '--thread-count=0',
 506                                        '--enable-reuse', "--collection-cache-size=256",
 507                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 508                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 509         expect_container["output_name"] = output_name
 510
 511         stubs.api.container_requests().create.assert_called_with(
 512             body=JsonDiffMatcher(expect_container))
 513         self.assertEqual(stubs.capture_stdout.getvalue(),
 514                          stubs.expect_container_request_uuid + '\n')
 515         self.assertEqual(exited, 0)
 516
 517     @stubs
 518     def test_submit_storage_classes(self, stubs):
 519         exited = arvados_cwl.main(
 520             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 521                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 522             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 523
 524         expect_container = copy.deepcopy(stubs.expect_container_spec)
 525         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 526                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 527                                        '--eval-timeout=20', '--thread-count=0',
 528                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 529                                        "--storage-classes=foo", '--on-error=continue',
 530                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 531
 532         stubs.api.container_requests().create.assert_called_with(
 533             body=JsonDiffMatcher(expect_container))
 534         self.assertEqual(stubs.capture_stdout.getvalue(),
 535                          stubs.expect_container_request_uuid + '\n')
 536         self.assertEqual(exited, 0)
 537
 538     @stubs
 539     def test_submit_multiple_storage_classes(self, stubs):
 540         exited = arvados_cwl.main(
 541             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 542                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 543             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 544
 545         expect_container = copy.deepcopy(stubs.expect_container_spec)
 546         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 547                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 548                                        '--eval-timeout=20', '--thread-count=0',
 549                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 550                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 551                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 552
 553         stubs.api.container_requests().create.assert_called_with(
 554             body=JsonDiffMatcher(expect_container))
 555         self.assertEqual(stubs.capture_stdout.getvalue(),
 556                          stubs.expect_container_request_uuid + '\n')
 557         self.assertEqual(exited, 0)
 558
 559     @mock.patch("cwltool.task_queue.TaskQueue")
 560     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 561     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 562     @stubs
 563     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 564         final_output_c = arvados.collection.Collection()
 565         make_output.return_value = ({},final_output_c)
 566
 567         def set_final_output(job_order, output_callback, runtimeContext):
 568             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 569             return []
 570         job.side_effect = set_final_output
 571
 572         exited = arvados_cwl.main(
 573             ["--debug", "--local", "--storage-classes=foo",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 578         self.assertEqual(exited, 0)
 579
 580     @mock.patch("cwltool.task_queue.TaskQueue")
 581     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 582     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 583     @stubs
 584     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 585         final_output_c = arvados.collection.Collection()
 586         make_output.return_value = ({},final_output_c)
 587         stubs.api.config().get.return_value = {"default": {"Default": True}}
 588
 589         def set_final_output(job_order, output_callback, runtimeContext):
 590             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 591             return []
 592         job.side_effect = set_final_output
 593
 594         exited = arvados_cwl.main(
 595             ["--debug", "--local",
 596                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598
 599         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 600         self.assertEqual(exited, 0)
 601
 602     @mock.patch("cwltool.task_queue.TaskQueue")
 603     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 604     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 605     @stubs
 606     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 607         final_output_c = arvados.collection.Collection()
 608         make_output.return_value = ({},final_output_c)
 609
 610         def set_final_output(job_order, output_callback, runtimeContext):
 611             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 612             return []
 613         job.side_effect = set_final_output
 614
 615         exited = arvados_cwl.main(
 616             ["--debug", "--local",
 617                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 618             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 619
 620         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 621         self.assertEqual(exited, 0)
 622
 623     @stubs
 624     def test_submit_container_output_ttl(self, stubs):
 625         exited = arvados_cwl.main(
 626             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 627                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 628             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 629
 630         expect_container = copy.deepcopy(stubs.expect_container_spec)
 631         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 632                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 633                                        '--eval-timeout=20', '--thread-count=0',
 634                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 635                                        '--on-error=continue',
 636                                        "--intermediate-output-ttl=3600",
 637                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 638
 639         stubs.api.container_requests().create.assert_called_with(
 640             body=JsonDiffMatcher(expect_container))
 641         self.assertEqual(stubs.capture_stdout.getvalue(),
 642                          stubs.expect_container_request_uuid + '\n')
 643         self.assertEqual(exited, 0)
 644
 645     @stubs
 646     def test_submit_container_trash_intermediate(self, stubs):
 647         exited = arvados_cwl.main(
 648             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 649                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 650             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 651
 652
 653         expect_container = copy.deepcopy(stubs.expect_container_spec)
 654         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 655                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 656                                        '--eval-timeout=20', '--thread-count=0',
 657                                        '--enable-reuse', "--collection-cache-size=256",
 658                                        '--debug', '--on-error=continue',
 659                                        "--trash-intermediate",
 660                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 661
 662         stubs.api.container_requests().create.assert_called_with(
 663             body=JsonDiffMatcher(expect_container))
 664         self.assertEqual(stubs.capture_stdout.getvalue(),
 665                          stubs.expect_container_request_uuid + '\n')
 666         self.assertEqual(exited, 0)
 667
 668     @stubs
 669     def test_submit_container_output_tags(self, stubs):
 670         output_tags = "tag0,tag1,tag2"
 671
 672         exited = arvados_cwl.main(
 673             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 674                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 676
 677         expect_container = copy.deepcopy(stubs.expect_container_spec)
 678         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 679                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 680                                        '--eval-timeout=20', '--thread-count=0',
 681                                        '--enable-reuse', "--collection-cache-size=256",
 682                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 683                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 684
 685         stubs.api.container_requests().create.assert_called_with(
 686             body=JsonDiffMatcher(expect_container))
 687         self.assertEqual(stubs.capture_stdout.getvalue(),
 688                          stubs.expect_container_request_uuid + '\n')
 689         self.assertEqual(exited, 0)
 690
 691     @stubs
 692     def test_submit_container_runner_ram(self, stubs):
 693         exited = arvados_cwl.main(
 694             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 695                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 696             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 697
 698         expect_container = copy.deepcopy(stubs.expect_container_spec)
 699         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 700
 701         stubs.api.container_requests().create.assert_called_with(
 702             body=JsonDiffMatcher(expect_container))
 703         self.assertEqual(stubs.capture_stdout.getvalue(),
 704                          stubs.expect_container_request_uuid + '\n')
 705         self.assertEqual(exited, 0)
 706
 707     @mock.patch("arvados.collection.CollectionReader")
 708     @mock.patch("time.sleep")
 709     @stubs
 710     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 711         collectionReader().exists.return_value = True
 712         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 713         exited = arvados_cwl.main(
 714             ["--submit", "--no-wait", "--api=containers", "--debug",
 715              "tests/wf/submit_keepref_wf.cwl"],
 716             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 717         self.assertEqual(exited, 0)
 718
 719     @mock.patch("arvados.collection.CollectionReader")
 720     @mock.patch("time.sleep")
 721     @stubs
 722     def test_submit_keepref(self, stubs, tm, reader):
 723         with open("tests/wf/expect_arvworkflow.cwl") as f:
 724             reader().open().__enter__().read.return_value = f.read()
 725
 726         exited = arvados_cwl.main(
 727             ["--submit", "--no-wait", "--api=containers", "--debug",
 728              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 729             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 730
 731         expect_container = {
 732             'priority': 500,
 733             'mounts': {
 734                 '/var/spool/cwl': {
 735                     'writable': True,
 736                     'kind': 'collection'
 737                 },
 738                 'stdout': {
 739                     'path': '/var/spool/cwl/cwl.output.json',
 740                     'kind': 'file'
 741                 },
 742                 '/var/lib/cwl/workflow': {
 743                     'portable_data_hash': '99999999999999999999999999999994+99',
 744                     'kind': 'collection'
 745                 },
 746                 '/var/lib/cwl/cwl.input.json': {
 747                     'content': {
 748                         'x': 'XxX'
 749                     },
 750                     'kind': 'json'
 751                 }
 752             }, 'state': 'Committed',
 753             'output_path': '/var/spool/cwl',
 754             'name': 'expect_arvworkflow.cwl#main',
 755             'container_image': '999999999999999999999999999999d3+99',
 756             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 757                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 758                         '--eval-timeout=20', '--thread-count=0',
 759                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 760                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 761             'cwd': '/var/spool/cwl',
 762             'runtime_constraints': {
 763                 'API': True,
 764                 'vcpus': 1,
 765                 'ram': 1342177280
 766             },
 767             'use_existing': False,
 768             'properties': {},
 769             'secret_mounts': {}
 770         }
 771
 772         stubs.api.container_requests().create.assert_called_with(
 773             body=JsonDiffMatcher(expect_container))
 774         self.assertEqual(stubs.capture_stdout.getvalue(),
 775                          stubs.expect_container_request_uuid + '\n')
 776         self.assertEqual(exited, 0)
 777
 778     @mock.patch("time.sleep")
 779     @stubs
 780     def test_submit_arvworkflow(self, stubs, tm):
 781         with open("tests/wf/expect_arvworkflow.cwl") as f:
 782             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 783
 784         exited = arvados_cwl.main(
 785             ["--submit", "--no-wait", "--api=containers", "--debug",
 786              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 787             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 788
 789         expect_container = {
 790             'priority': 500,
 791             'mounts': {
 792                 '/var/spool/cwl': {
 793                     'writable': True,
 794                     'kind': 'collection'
 795                 },
 796                 'stdout': {
 797                     'path': '/var/spool/cwl/cwl.output.json',
 798                     'kind': 'file'
 799                 },
 800                 '/var/lib/cwl/workflow.json': {
 801                     'kind': 'json',
 802                     'content': {
 803                         'cwlVersion': 'v1.0',
 804                         '$graph': [
 805                             {
 806                                 'id': '#main',
 807                                 'inputs': [
 808                                     {'type': 'string', 'id': '#main/x'}
 809                                 ],
 810                                 'steps': [
 811                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 812                                      'run': '#submit_tool.cwl',
 813                                      'id': '#main/step1',
 814                                      'out': []}
 815                                 ],
 816                                 'class': 'Workflow',
 817                                 'outputs': []
 818                             },
 819                             {
 820                                 'inputs': [
 821                                     {
 822                                         'inputBinding': {'position': 1},
 823                                         'type': 'string',
 824                                         'id': '#submit_tool.cwl/x'}
 825                                 ],
 826                                 'requirements': [
 827                                     {
 828                                         'dockerPull': 'debian:buster-slim',
 829                                         'class': 'DockerRequirement',
 830                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 831                                     }
 832                                 ],
 833                                 'id': '#submit_tool.cwl',
 834                                 'outputs': [],
 835                                 'baseCommand': 'cat',
 836                                 'class': 'CommandLineTool'
 837                             }
 838                         ]
 839                     }
 840                 },
 841                 '/var/lib/cwl/cwl.input.json': {
 842                     'content': {
 843                         'x': 'XxX'
 844                     },
 845                     'kind': 'json'
 846                 }
 847             }, 'state': 'Committed',
 848             'output_path': '/var/spool/cwl',
 849             'name': 'a test workflow',
 850             'container_image': "999999999999999999999999999999d3+99",
 851             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 852                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 853                         '--eval-timeout=20', '--thread-count=0',
 854                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 855                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 856             'cwd': '/var/spool/cwl',
 857             'runtime_constraints': {
 858                 'API': True,
 859                 'vcpus': 1,
 860                 'ram': 1342177280
 861             },
 862             'use_existing': False,
 863             'properties': {
 864                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 865             },
 866             'secret_mounts': {}
 867         }
 868
 869         stubs.api.container_requests().create.assert_called_with(
 870             body=JsonDiffMatcher(expect_container))
 871         self.assertEqual(stubs.capture_stdout.getvalue(),
 872                          stubs.expect_container_request_uuid + '\n')
 873         self.assertEqual(exited, 0)
 874
 875     @stubs
 876     def test_submit_container_name(self, stubs):
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 879                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["name"] = "hello container 123"
 884
 885         stubs.api.container_requests().create.assert_called_with(
 886             body=JsonDiffMatcher(expect_container))
 887         self.assertEqual(stubs.capture_stdout.getvalue(),
 888                          stubs.expect_container_request_uuid + '\n')
 889         self.assertEqual(exited, 0)
 890
 891     @stubs
 892     def test_submit_missing_input(self, stubs):
 893         exited = arvados_cwl.main(
 894             ["--submit", "--no-wait", "--api=containers", "--debug",
 895              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 896             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 897         self.assertEqual(exited, 0)
 898
 899         exited = arvados_cwl.main(
 900             ["--submit", "--no-wait", "--api=containers", "--debug",
 901              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 902             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 903         self.assertEqual(exited, 1)
 904
 905     @stubs
 906     def test_submit_container_project(self, stubs):
 907         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 908         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 909         exited = arvados_cwl.main(
 910             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 911                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 912             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 913
 914         expect_container = copy.deepcopy(stubs.expect_container_spec)
 915         expect_container["owner_uuid"] = project_uuid
 916         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 917                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 918                                        "--eval-timeout=20", "--thread-count=0",
 919                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 920                                        '--on-error=continue',
 921                                        '--project-uuid='+project_uuid,
 922                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 923
 924         stubs.api.container_requests().create.assert_called_with(
 925             body=JsonDiffMatcher(expect_container))
 926         self.assertEqual(stubs.capture_stdout.getvalue(),
 927                          stubs.expect_container_request_uuid + '\n')
 928         self.assertEqual(exited, 0)
 929
 930     @stubs
 931     def test_submit_container_eval_timeout(self, stubs):
 932         exited = arvados_cwl.main(
 933             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 934                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 935             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 936
 937         expect_container = copy.deepcopy(stubs.expect_container_spec)
 938         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 939                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 940                                        '--eval-timeout=60.0', '--thread-count=0',
 941                                        '--enable-reuse', "--collection-cache-size=256",
 942                                        '--debug', '--on-error=continue',
 943                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 944
 945         stubs.api.container_requests().create.assert_called_with(
 946             body=JsonDiffMatcher(expect_container))
 947         self.assertEqual(stubs.capture_stdout.getvalue(),
 948                          stubs.expect_container_request_uuid + '\n')
 949         self.assertEqual(exited, 0)
 950
 951     @stubs
 952     def test_submit_container_collection_cache(self, stubs):
 953         exited = arvados_cwl.main(
 954             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 955                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 956             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 957
 958         expect_container = copy.deepcopy(stubs.expect_container_spec)
 959         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 960                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 961                                        '--eval-timeout=20', '--thread-count=0',
 962                                        '--enable-reuse', "--collection-cache-size=500",
 963                                        '--debug', '--on-error=continue',
 964                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 965         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 966
 967         stubs.api.container_requests().create.assert_called_with(
 968             body=JsonDiffMatcher(expect_container))
 969         self.assertEqual(stubs.capture_stdout.getvalue(),
 970                          stubs.expect_container_request_uuid + '\n')
 971         self.assertEqual(exited, 0)
 972
 973     @stubs
 974     def test_submit_container_thread_count(self, stubs):
 975         exited = arvados_cwl.main(
 976             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 977                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 978             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 979
 980         expect_container = copy.deepcopy(stubs.expect_container_spec)
 981         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 982                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 983                                        '--eval-timeout=20', '--thread-count=20',
 984                                        '--enable-reuse', "--collection-cache-size=256",
 985                                        '--debug', '--on-error=continue',
 986                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 987
 988         stubs.api.container_requests().create.assert_called_with(
 989             body=JsonDiffMatcher(expect_container))
 990         self.assertEqual(stubs.capture_stdout.getvalue(),
 991                          stubs.expect_container_request_uuid + '\n')
 992         self.assertEqual(exited, 0)
 993
 994     @stubs
 995     def test_submit_container_runner_image(self, stubs):
 996         exited = arvados_cwl.main(
 997             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 998                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 999             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1000
1001         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1002
1003         expect_container = copy.deepcopy(stubs.expect_container_spec)
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     @stubs
1011     def test_submit_priority(self, stubs):
1012         exited = arvados_cwl.main(
1013             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1014                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1015             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1016
1017         stubs.expect_container_spec["priority"] = 669
1018
1019         expect_container = copy.deepcopy(stubs.expect_container_spec)
1020         stubs.api.container_requests().create.assert_called_with(
1021             body=JsonDiffMatcher(expect_container))
1022         self.assertEqual(stubs.capture_stdout.getvalue(),
1023                          stubs.expect_container_request_uuid + '\n')
1024         self.assertEqual(exited, 0)
1025
1026     @stubs
1027     def test_submit_wf_runner_resources(self, stubs):
1028         exited = arvados_cwl.main(
1029             ["--submit", "--no-wait", "--api=containers", "--debug",
1030                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1031             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1032
1033         expect_container = copy.deepcopy(stubs.expect_container_spec)
1034         expect_container["runtime_constraints"] = {
1035             "API": True,
1036             "vcpus": 2,
1037             "ram": (2000+512) * 2**20
1038         }
1039         expect_container["name"] = "submit_wf_runner_resources.cwl"
1040         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1041             {
1042                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1043                 "coresMin": 2,
1044                 "ramMin": 2000,
1045                 "keep_cache": 512
1046             }
1047         ]
1048         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1049             "arv": "http://arvados.org/cwl#",
1050         }
1051         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1052                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1053                         '--eval-timeout=20', '--thread-count=0',
1054                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1055                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1056
1057         stubs.api.container_requests().create.assert_called_with(
1058             body=JsonDiffMatcher(expect_container))
1059         self.assertEqual(stubs.capture_stdout.getvalue(),
1060                          stubs.expect_container_request_uuid + '\n')
1061         self.assertEqual(exited, 0)
1062
1063     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1064     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1065     @mock.patch("arvados.api")
1066     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1067         arvados_cwl.arvdocker.arv_docker_clear_cache()
1068
1069         arvrunner = mock.MagicMock()
1070         arvrunner.project_uuid = ""
1071         api.return_value = mock.MagicMock()
1072         arvrunner.api = api.return_value
1073         arvrunner.runtimeContext.match_local_docker = False
1074         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1075                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1076                                                                         "link_class": "docker_image_repo+tag",
1077                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1078                                                                         "owner_uuid": "",
1079                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1080                                                             {"items": [{"created_at": "",
1081                                                                         "head_uuid": "",
1082                                                                         "link_class": "docker_image_hash",
1083                                                                         "name": "123456",
1084                                                                         "owner_uuid": "",
1085                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1086                                                             {"items": [{"created_at": "",
1087                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1088                                                                         "link_class": "docker_image_repo+tag",
1089                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1090                                                                         "owner_uuid": "",
1091                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1092                                                             {"items": [{"created_at": "",
1093                                                                         "head_uuid": "",
1094                                                                         "link_class": "docker_image_hash",
1095                                                                         "name": "123456",
1096                                                                         "owner_uuid": "",
1097                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1098         )
1099         find_one_image_hash.return_value = "123456"
1100
1101         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1102                                                                               "owner_uuid": "",
1103                                                                               "manifest_text": "",
1104                                                                               "properties": ""
1105                                                                               }], "items_available": 1, "offset": 0},
1106                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1107                                                                               "owner_uuid": "",
1108                                                                               "manifest_text": "",
1109                                                                               "properties": ""
1110                                                                           }], "items_available": 1, "offset": 0})
1111         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1112         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1113                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1114
1115         self.assertEqual("9999999999999999999999999999999b+99",
1116                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1117
1118
1119     @stubs
1120     def test_submit_secrets(self, stubs):
1121         exited = arvados_cwl.main(
1122             ["--submit", "--no-wait", "--api=containers", "--debug",
1123                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1124             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1125
1126         expect_container = {
1127             "command": [
1128                 "arvados-cwl-runner",
1129                 "--local",
1130                 "--api=containers",
1131                 "--no-log-timestamps",
1132                 "--disable-validate",
1133                 "--disable-color",
1134                 "--eval-timeout=20",
1135                 '--thread-count=0',
1136                 "--enable-reuse",
1137                 "--collection-cache-size=256",
1138                 '--debug',
1139                 "--on-error=continue",
1140                 "/var/lib/cwl/workflow.json#main",
1141                 "/var/lib/cwl/cwl.input.json"
1142             ],
1143             "container_image": "999999999999999999999999999999d3+99",
1144             "cwd": "/var/spool/cwl",
1145             "mounts": {
1146                 "/var/lib/cwl/cwl.input.json": {
1147                     "content": {
1148                         "pw": {
1149                             "$include": "/secrets/s0"
1150                         }
1151                     },
1152                     "kind": "json"
1153                 },
1154                 "/var/lib/cwl/workflow.json": {
1155                     "content": {
1156                         "$graph": [
1157                             {
1158                                 "arguments": [
1159                                     "md5sum",
1160                                     "example.conf"
1161                                 ],
1162                                 "class": "CommandLineTool",
1163                                 "hints": [
1164                                     {
1165                                         "class": "http://commonwl.org/cwltool#Secrets",
1166                                         "secrets": [
1167                                             "#secret_job.cwl/pw"
1168                                         ]
1169                                     }
1170                                 ],
1171                                 "id": "#secret_job.cwl",
1172                                 "inputs": [
1173                                     {
1174                                         "id": "#secret_job.cwl/pw",
1175                                         "type": "string"
1176                                     }
1177                                 ],
1178                                 "outputs": [
1179                                     {
1180                                         "id": "#secret_job.cwl/out",
1181                                         "type": "File",
1182                                         "outputBinding": {
1183                                               "glob": "hashed_example.txt"
1184                                         }
1185                                     }
1186                                 ],
1187                                 "stdout": "hashed_example.txt",
1188                                 "requirements": [
1189                                     {
1190                                         "class": "InitialWorkDirRequirement",
1191                                         "listing": [
1192                                             {
1193                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1194                                                 "entryname": "example.conf"
1195                                             }
1196                                         ]
1197                                     }
1198                                 ]
1199                             },
1200                             {
1201                                 "class": "Workflow",
1202                                 "hints": [
1203                                     {
1204                                         "class": "DockerRequirement",
1205                                         "dockerPull": "debian:buster-slim",
1206                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1207                                     },
1208                                     {
1209                                         "class": "http://commonwl.org/cwltool#Secrets",
1210                                         "secrets": [
1211                                             "#main/pw"
1212                                         ]
1213                                     }
1214                                 ],
1215                                 "id": "#main",
1216                                 "inputs": [
1217                                     {
1218                                         "id": "#main/pw",
1219                                         "type": "string"
1220                                     }
1221                                 ],
1222                                 "outputs": [
1223                                     {
1224                                         "id": "#main/out",
1225                                         "outputSource": "#main/step1/out",
1226                                         "type": "File"
1227                                     }
1228                                 ],
1229                                 "steps": [
1230                                     {
1231                                         "id": "#main/step1",
1232                                         "in": [
1233                                             {
1234                                                 "id": "#main/step1/pw",
1235                                                 "source": "#main/pw"
1236                                             }
1237                                         ],
1238                                         "out": [
1239                                             "#main/step1/out"
1240                                         ],
1241                                         "run": "#secret_job.cwl"
1242                                     }
1243                                 ]
1244                             }
1245                         ],
1246                         "$namespaces": {
1247                             "cwltool": "http://commonwl.org/cwltool#"
1248                         },
1249                         "cwlVersion": "v1.0"
1250                     },
1251                     "kind": "json"
1252                 },
1253                 "/var/spool/cwl": {
1254                     "kind": "collection",
1255                     "writable": True
1256                 },
1257                 "stdout": {
1258                     "kind": "file",
1259                     "path": "/var/spool/cwl/cwl.output.json"
1260                 }
1261             },
1262             "name": "secret_wf.cwl",
1263             "output_path": "/var/spool/cwl",
1264             "priority": 500,
1265             "properties": {},
1266             "runtime_constraints": {
1267                 "API": True,
1268                 "ram": 1342177280,
1269                 "vcpus": 1
1270             },
1271             "secret_mounts": {
1272                 "/secrets/s0": {
1273                     "content": "blorp",
1274                     "kind": "text"
1275                 }
1276             },
1277             "state": "Committed",
1278             "use_existing": False
1279         }
1280
1281         stubs.api.container_requests().create.assert_called_with(
1282             body=JsonDiffMatcher(expect_container))
1283         self.assertEqual(stubs.capture_stdout.getvalue(),
1284                          stubs.expect_container_request_uuid + '\n')
1285         self.assertEqual(exited, 0)
1286
1287     @stubs
1288     def test_submit_request_uuid(self, stubs):
1289         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1290         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1291
1292         stubs.api.container_requests().update().execute.return_value = {
1293             "uuid": stubs.expect_container_request_uuid,
1294             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1295             "state": "Queued"
1296         }
1297
1298         exited = arvados_cwl.main(
1299             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1300                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1301             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1302
1303         stubs.api.container_requests().update.assert_called_with(
1304             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1305         self.assertEqual(stubs.capture_stdout.getvalue(),
1306                          stubs.expect_container_request_uuid + '\n')
1307         self.assertEqual(exited, 0)
1308
1309     @stubs
1310     def test_submit_container_cluster_id(self, stubs):
1311         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1312
1313         exited = arvados_cwl.main(
1314             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1315                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1316             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1317
1318         expect_container = copy.deepcopy(stubs.expect_container_spec)
1319
1320         stubs.api.container_requests().create.assert_called_with(
1321             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1322         self.assertEqual(stubs.capture_stdout.getvalue(),
1323                          stubs.expect_container_request_uuid + '\n')
1324         self.assertEqual(exited, 0)
1325
1326     @stubs
1327     def test_submit_validate_cluster_id(self, stubs):
1328         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1329         exited = arvados_cwl.main(
1330             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1331              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1332             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1333         self.assertEqual(exited, 1)
1334
1335     @stubs
1336     def test_submit_validate_project_uuid(self, stubs):
1337         # Fails with bad cluster prefix
1338         exited = arvados_cwl.main(
1339             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1340              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1341             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1342         self.assertEqual(exited, 1)
1343
1344         # Project lookup fails
1345         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1346         exited = arvados_cwl.main(
1347             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1348              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1349             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1350         self.assertEqual(exited, 1)
1351
1352         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1353         exited = arvados_cwl.main(
1354             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1355              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1356             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1357         self.assertEqual(exited, 0)
1358
1359
1360     @mock.patch("arvados.collection.CollectionReader")
1361     @stubs
1362     def test_submit_uuid_inputs(self, stubs, collectionReader):
1363         collectionReader().exists.return_value = True
1364         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1365         def list_side_effect(**kwargs):
1366             m = mock.MagicMock()
1367             if "count" in kwargs:
1368                 m.execute.return_value = {"items": [
1369                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1370                 ]}
1371             else:
1372                 m.execute.return_value = {"items": []}
1373             return m
1374         stubs.api.collections().list.side_effect = list_side_effect
1375
1376         exited = arvados_cwl.main(
1377             ["--submit", "--no-wait", "--api=containers", "--debug",
1378                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1379             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1380
1381         expect_container = copy.deepcopy(stubs.expect_container_spec)
1382         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1383         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1384         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1385
1386         stubs.api.collections().list.assert_has_calls([
1387             mock.call(count='none',
1388                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1389                       select=['uuid', 'portable_data_hash'])])
1390         stubs.api.container_requests().create.assert_called_with(
1391             body=JsonDiffMatcher(expect_container))
1392         self.assertEqual(stubs.capture_stdout.getvalue(),
1393                          stubs.expect_container_request_uuid + '\n')
1394         self.assertEqual(exited, 0)
1395
1396     @stubs
1397     def test_submit_mismatched_uuid_inputs(self, stubs):
1398         def list_side_effect(**kwargs):
1399             m = mock.MagicMock()
1400             if "count" in kwargs:
1401                 m.execute.return_value = {"items": [
1402                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1403                 ]}
1404             else:
1405                 m.execute.return_value = {"items": []}
1406             return m
1407         stubs.api.collections().list.side_effect = list_side_effect
1408
1409         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1410             capture_stderr = StringIO()
1411             cwltool_logger = logging.getLogger('cwltool')
1412             stderr_logger = logging.StreamHandler(capture_stderr)
1413             cwltool_logger.addHandler(stderr_logger)
1414
1415             try:
1416                 exited = arvados_cwl.main(
1417                     ["--submit", "--no-wait", "--api=containers", "--debug",
1418                         "tests/wf/submit_wf.cwl", infile],
1419                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1420
1421                 self.assertEqual(exited, 1)
1422                 self.assertRegex(
1423                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1424                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1425             finally:
1426                 cwltool_logger.removeHandler(stderr_logger)
1427
1428     @mock.patch("arvados.collection.CollectionReader")
1429     @stubs
1430     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1431         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1432         capture_stderr = StringIO()
1433
1434         cwltool_logger = logging.getLogger('cwltool')
1435         stderr_logger = logging.StreamHandler(capture_stderr)
1436         cwltool_logger.addHandler(stderr_logger)
1437
1438         exited = arvados_cwl.main(
1439             ["--submit", "--no-wait", "--api=containers", "--debug",
1440                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1441             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1442
1443         try:
1444             self.assertEqual(exited, 1)
1445             self.assertRegex(
1446                 capture_stderr.getvalue(),
1447                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1448         finally:
1449             cwltool_logger.removeHandler(stderr_logger)
1450
1451     @stubs
1452     def test_submit_set_process_properties(self, stubs):
1453         exited = arvados_cwl.main(
1454             ["--submit", "--no-wait", "--api=containers", "--debug",
1455                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1456             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1457
1458         expect_container = copy.deepcopy(stubs.expect_container_spec)
1459         expect_container["name"] = "submit_wf_process_properties.cwl"
1460         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1461             {
1462                 "class": "http://arvados.org/cwl#ProcessProperties",
1463                 "processProperties": [
1464                     {"propertyName": "baz",
1465                      "propertyValue": "$(inputs.x.basename)"},
1466                     {"propertyName": "foo",
1467                      "propertyValue": "bar"},
1468                     {"propertyName": "quux",
1469                      "propertyValue": {
1470                          "q1": 1,
1471                          "q2": 2
1472                      }
1473                     }
1474                 ],
1475             }
1476         ]
1477         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1478             "arv": "http://arvados.org/cwl#"
1479         }
1480
1481         expect_container["properties"] = {
1482             "baz": "blorp.txt",
1483             "foo": "bar",
1484             "quux": {
1485                 "q1": 1,
1486                 "q2": 2
1487             }
1488         }
1489
1490         stubs.api.container_requests().create.assert_called_with(
1491             body=JsonDiffMatcher(expect_container))
1492         self.assertEqual(stubs.capture_stdout.getvalue(),
1493                          stubs.expect_container_request_uuid + '\n')
1494         self.assertEqual(exited, 0)
1495
1496
1497     @stubs
1498     def test_submit_enable_preemptible(self, stubs):
1499         exited = arvados_cwl.main(
1500             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1501                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1502             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1503
1504         expect_container = copy.deepcopy(stubs.expect_container_spec)
1505         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1506                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1507                         '--eval-timeout=20', '--thread-count=0',
1508                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1509                                        '--enable-preemptible',
1510                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1511
1512         stubs.api.container_requests().create.assert_called_with(
1513             body=JsonDiffMatcher(expect_container))
1514         self.assertEqual(stubs.capture_stdout.getvalue(),
1515                          stubs.expect_container_request_uuid + '\n')
1516         self.assertEqual(exited, 0)
1517
1518     @stubs
1519     def test_submit_disable_preemptible(self, stubs):
1520         exited = arvados_cwl.main(
1521             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1522                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1523             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1524
1525         expect_container = copy.deepcopy(stubs.expect_container_spec)
1526         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1527                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1528                         '--eval-timeout=20', '--thread-count=0',
1529                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1530                                        '--disable-preemptible',
1531                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1532
1533         stubs.api.container_requests().create.assert_called_with(
1534             body=JsonDiffMatcher(expect_container))
1535         self.assertEqual(stubs.capture_stdout.getvalue(),
1536                          stubs.expect_container_request_uuid + '\n')
1537         self.assertEqual(exited, 0)
1538
1539
1540 class TestCreateWorkflow(unittest.TestCase):
1541     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1542     expect_workflow = StripYAMLComments(
1543         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1544
1545     def setUp(self):
1546         cwltool.process._names = set()
1547         arvados_cwl.arvdocker.arv_docker_clear_cache()
1548
1549     def tearDown(self):
1550         root_logger = logging.getLogger('')
1551
1552         # Remove existing RuntimeStatusLoggingHandlers if they exist
1553         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1554         root_logger.handlers = handlers
1555
1556     @stubs
1557     def test_create(self, stubs):
1558         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1559         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1560
1561         exited = arvados_cwl.main(
1562             ["--create-workflow", "--debug",
1563              "--api=containers",
1564              "--project-uuid", project_uuid,
1565              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1566             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1567
1568         stubs.api.pipeline_templates().create.refute_called()
1569         stubs.api.container_requests().create.refute_called()
1570
1571         body = {
1572             "workflow": {
1573                 "owner_uuid": project_uuid,
1574                 "name": "submit_wf.cwl",
1575                 "description": "",
1576                 "definition": self.expect_workflow,
1577             }
1578         }
1579         stubs.api.workflows().create.assert_called_with(
1580             body=JsonDiffMatcher(body))
1581
1582         self.assertEqual(stubs.capture_stdout.getvalue(),
1583                          stubs.expect_workflow_uuid + '\n')
1584         self.assertEqual(exited, 0)
1585
1586     @stubs
1587     def test_create_name(self, stubs):
1588         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1589         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1590
1591         exited = arvados_cwl.main(
1592             ["--create-workflow", "--debug",
1593              "--api=containers",
1594              "--project-uuid", project_uuid,
1595              "--name", "testing 123",
1596              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1597             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1598
1599         stubs.api.pipeline_templates().create.refute_called()
1600         stubs.api.container_requests().create.refute_called()
1601
1602         body = {
1603             "workflow": {
1604                 "owner_uuid": project_uuid,
1605                 "name": "testing 123",
1606                 "description": "",
1607                 "definition": self.expect_workflow,
1608             }
1609         }
1610         stubs.api.workflows().create.assert_called_with(
1611             body=JsonDiffMatcher(body))
1612
1613         self.assertEqual(stubs.capture_stdout.getvalue(),
1614                          stubs.expect_workflow_uuid + '\n')
1615         self.assertEqual(exited, 0)
1616
1617
1618     @stubs
1619     def test_update(self, stubs):
1620         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1621         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1622
1623         exited = arvados_cwl.main(
1624             ["--update-workflow", self.existing_workflow_uuid,
1625              "--debug",
1626              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1627             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1628
1629         body = {
1630             "workflow": {
1631                 "name": "submit_wf.cwl",
1632                 "description": "",
1633                 "definition": self.expect_workflow,
1634                 "owner_uuid": project_uuid
1635             }
1636         }
1637         stubs.api.workflows().update.assert_called_with(
1638             uuid=self.existing_workflow_uuid,
1639             body=JsonDiffMatcher(body))
1640         self.assertEqual(stubs.capture_stdout.getvalue(),
1641                          self.existing_workflow_uuid + '\n')
1642         self.assertEqual(exited, 0)
1643
1644
1645     @stubs
1646     def test_update_name(self, stubs):
1647         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1648         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1649
1650         exited = arvados_cwl.main(
1651             ["--update-workflow", self.existing_workflow_uuid,
1652              "--debug", "--name", "testing 123",
1653              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1654             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1655
1656         body = {
1657             "workflow": {
1658                 "name": "testing 123",
1659                 "description": "",
1660                 "definition": self.expect_workflow,
1661                 "owner_uuid": project_uuid
1662             }
1663         }
1664         stubs.api.workflows().update.assert_called_with(
1665             uuid=self.existing_workflow_uuid,
1666             body=JsonDiffMatcher(body))
1667         self.assertEqual(stubs.capture_stdout.getvalue(),
1668                          self.existing_workflow_uuid + '\n')
1669         self.assertEqual(exited, 0)
1670
1671     @stubs
1672     def test_create_collection_per_tool(self, stubs):
1673         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1674         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1675
1676         exited = arvados_cwl.main(
1677             ["--create-workflow", "--debug",
1678              "--api=containers",
1679              "--project-uuid", project_uuid,
1680              "tests/collection_per_tool/collection_per_tool.cwl"],
1681             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1682
1683         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1684         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1685
1686         body = {
1687             "workflow": {
1688                 "owner_uuid": project_uuid,
1689                 "name": "collection_per_tool.cwl",
1690                 "description": "",
1691                 "definition": expect_workflow,
1692             }
1693         }
1694         stubs.api.workflows().create.assert_called_with(
1695             body=JsonDiffMatcher(body))
1696
1697         self.assertEqual(stubs.capture_stdout.getvalue(),
1698                          stubs.expect_workflow_uuid + '\n')
1699         self.assertEqual(exited, 0)
1700
1701     @stubs
1702     def test_create_with_imports(self, stubs):
1703         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1704         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1705
1706         exited = arvados_cwl.main(
1707             ["--create-workflow", "--debug",
1708              "--api=containers",
1709              "--project-uuid", project_uuid,
1710              "tests/wf/feddemo/feddemo.cwl"],
1711             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1712
1713         stubs.api.pipeline_templates().create.refute_called()
1714         stubs.api.container_requests().create.refute_called()
1715
1716         self.assertEqual(stubs.capture_stdout.getvalue(),
1717                          stubs.expect_workflow_uuid + '\n')
1718         self.assertEqual(exited, 0)
1719
1720     @stubs
1721     def test_create_with_no_input(self, stubs):
1722         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1723         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1724
1725         exited = arvados_cwl.main(
1726             ["--create-workflow", "--debug",
1727              "--api=containers",
1728              "--project-uuid", project_uuid,
1729              "tests/wf/revsort/revsort.cwl"],
1730             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1731
1732         stubs.api.pipeline_templates().create.refute_called()
1733         stubs.api.container_requests().create.refute_called()
1734
1735         self.assertEqual(stubs.capture_stdout.getvalue(),
1736                          stubs.expect_workflow_uuid + '\n')
1737         self.assertEqual(exited, 0)