sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("uuid.uuid4")
  51     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  52     @mock.patch("arvados.collection.KeepClient")
  53     @mock.patch("arvados.keep.KeepClient")
  54     @mock.patch("arvados.events.subscribe")
  55     def wrapped(self, events, keep_client1, keep_client2, keepdocker, uuid4, *args, **kwargs):
  56         class Stubs(object):
  57             pass
  58         stubs = Stubs()
  59         stubs.events = events
  60         stubs.keepdocker = keepdocker
  61
  62         uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  63                              "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  64                              "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  65
  66         def putstub(p, **kwargs):
  67             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  68         keep_client1().put.side_effect = putstub
  69         keep_client1.put.side_effect = putstub
  70         keep_client2().put.side_effect = putstub
  71         keep_client2.put.side_effect = putstub
  72
  73         stubs.keep_client = keep_client2
  74         stubs.docker_images = {
  75             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  76             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  77             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  78             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  79         }
  80         def kd(a, b, image_name=None, image_tag=None):
  81             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  82         stubs.keepdocker.side_effect = kd
  83
  84         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  85         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  86
  87         if sys.version_info[0] < 3:
  88             stubs.capture_stdout = BytesIO()
  89         else:
  90             stubs.capture_stdout = StringIO()
  91
  92         stubs.api = mock.MagicMock()
  93         stubs.api._rootDesc = get_rootDesc()
  94         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  95         stubs.api._rootDesc["revision"] = "20210628"
  96
  97         stubs.api.users().current().execute.return_value = {
  98             "uuid": stubs.fake_user_uuid,
  99         }
 100         stubs.api.collections().list().execute.return_value = {"items": []}
 101         stubs.api.containers().current().execute.return_value = {
 102             "uuid": stubs.fake_container_uuid,
 103         }
 104         stubs.api.config()["StorageClasses"].items.return_value = {
 105             "default": {
 106                 "Default": True
 107             }
 108         }.items()
 109
 110         class CollectionExecute(object):
 111             def __init__(self, exe):
 112                 self.exe = exe
 113             def execute(self, num_retries=None):
 114                 return self.exe
 115
 116         def collection_createstub(created_collections, body, ensure_unique_name=None):
 117             mt = body["manifest_text"].encode('utf-8')
 118             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 119             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 120             created_collections[uuid] = {
 121                 "uuid": uuid,
 122                 "portable_data_hash": pdh,
 123                 "manifest_text": mt.decode('utf-8')
 124             }
 125             return CollectionExecute(created_collections[uuid])
 126
 127         def collection_getstub(created_collections, uuid):
 128             for v in viewvalues(created_collections):
 129                 if uuid in (v["uuid"], v["portable_data_hash"]):
 130                     return CollectionExecute(v)
 131
 132         created_collections = {
 133             "99999999999999999999999999999998+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999998+99",
 136                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999997+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999997+99",
 141                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 142             },
 143             "99999999999999999999999999999994+99": {
 144                 "uuid": "",
 145                 "portable_data_hash": "99999999999999999999999999999994+99",
 146                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 150                 "portable_data_hash": "999999999999999999999999999999d3+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 155                 "portable_data_hash": "999999999999999999999999999999d4+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 160                 "portable_data_hash": "999999999999999999999999999999d5+99",
 161                 "manifest_text": ""
 162             },
 163             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 164                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 165                 "portable_data_hash": "999999999999999999999999999999d6+99",
 166                 "manifest_text": ""
 167             }
 168         }
 169         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 170         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 171
 172         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 173         stubs.api.jobs().create().execute.return_value = {
 174             "uuid": stubs.expect_job_uuid,
 175             "state": "Queued",
 176         }
 177
 178         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 179         stubs.api.container_requests().create().execute.return_value = {
 180             "uuid": stubs.expect_container_request_uuid,
 181             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 182             "state": "Queued"
 183         }
 184
 185         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 186         stubs.api.pipeline_templates().create().execute.return_value = {
 187             "uuid": stubs.expect_pipeline_template_uuid,
 188         }
 189         stubs.expect_job_spec = {
 190             'runtime_constraints': {
 191                 'docker_image': '999999999999999999999999999999d3+99',
 192                 'min_ram_mb_per_node': 1024
 193             },
 194             'script_parameters': {
 195                 'x': {
 196                     'basename': 'blorp.txt',
 197                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 198                     'class': 'File'
 199                 },
 200                 'y': {
 201                     'basename': '99999999999999999999999999999998+99',
 202                     'location': 'keep:99999999999999999999999999999998+99',
 203                     'class': 'Directory'
 204                 },
 205                 'z': {
 206                     'basename': 'anonymous',
 207                     "listing": [{
 208                         "basename": "renamed.txt",
 209                         "class": "File",
 210                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 211                         "size": 0
 212                     }],
 213                     'class': 'Directory'
 214                 },
 215                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 216             },
 217             'repository': 'arvados',
 218             'script_version': 'master',
 219             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 220             'script': 'cwl-runner'
 221         }
 222         stubs.pipeline_component = stubs.expect_job_spec.copy()
 223         stubs.expect_pipeline_instance = {
 224             'name': 'submit_wf.cwl',
 225             'state': 'RunningOnServer',
 226             'owner_uuid': None,
 227             "components": {
 228                 "cwl-runner": {
 229                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 230                     'script_parameters': {
 231                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 232                         'x': {"value": {
 233                             'basename': 'blorp.txt',
 234                             'class': 'File',
 235                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 236                             "size": 16
 237                         }},
 238                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 239                               'listing': [
 240                                   {
 241                                       'basename': 'renamed.txt',
 242                                       'class': 'File', 'location':
 243                                       'keep:99999999999999999999999999999998+99/file1.txt',
 244                                       'size': 0
 245                                   }
 246                               ]}},
 247                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 248                         'arv:debug': True,
 249                         'arv:enable_reuse': True,
 250                         'arv:on_error': 'continue'
 251                     },
 252                     'repository': 'arvados',
 253                     'script_version': 'master',
 254                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 255                     'script': 'cwl-runner',
 256                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 257                 }
 258             }
 259         }
 260         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 261         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 262         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 263         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 264         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 265             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 266             "state": "Queued"
 267         }
 268         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 269         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 270
 271         with open("tests/wf/submit_wf_packed.cwl") as f:
 272             expect_packed_workflow = yaml.round_trip_load(f)
 273
 274         stubs.expect_container_spec = {
 275             'priority': 500,
 276             'mounts': {
 277                 '/var/spool/cwl': {
 278                     'writable': True,
 279                     'kind': 'collection'
 280                 },
 281                 '/var/lib/cwl/workflow.json': {
 282                     'content': expect_packed_workflow,
 283                     'kind': 'json'
 284                 },
 285                 'stdout': {
 286                     'path': '/var/spool/cwl/cwl.output.json',
 287                     'kind': 'file'
 288                 },
 289                 '/var/lib/cwl/cwl.input.json': {
 290                     'kind': 'json',
 291                     'content': {
 292                         'y': {
 293                             'basename': '99999999999999999999999999999998+99',
 294                             'location': 'keep:99999999999999999999999999999998+99',
 295                             'class': 'Directory'},
 296                         'x': {
 297                             'basename': u'blorp.txt',
 298                             'class': 'File',
 299                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 300                             "size": 16
 301                         },
 302                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 303                             {'basename': 'renamed.txt',
 304                              'class': 'File',
 305                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 306                              'size': 0
 307                             }
 308                         ]}
 309                     },
 310                     'kind': 'json'
 311                 }
 312             },
 313             'secret_mounts': {},
 314             'state': 'Committed',
 315             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 316                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 317                         '--eval-timeout=20', '--thread-count=0',
 318                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 319                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 320             'name': 'submit_wf.cwl',
 321             'container_image': '999999999999999999999999999999d3+99',
 322             'output_path': '/var/spool/cwl',
 323             'cwd': '/var/spool/cwl',
 324             'runtime_constraints': {
 325                 'API': True,
 326                 'vcpus': 1,
 327                 'ram': (1024+256)*1024*1024
 328             },
 329             'use_existing': False,
 330             'properties': {},
 331             'secret_mounts': {}
 332         }
 333
 334         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 335         stubs.api.workflows().create().execute.return_value = {
 336             "uuid": stubs.expect_workflow_uuid,
 337         }
 338         def update_mock(**kwargs):
 339             stubs.updated_uuid = kwargs.get('uuid')
 340             return mock.DEFAULT
 341         stubs.api.workflows().update.side_effect = update_mock
 342         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 343             "uuid": stubs.updated_uuid,
 344         }
 345
 346         return func(self, stubs, *args, **kwargs)
 347     return wrapped
 348
 349
 350 class TestSubmit(unittest.TestCase):
 351
 352     def setUp(self):
 353         cwltool.process._names = set()
 354         arvados_cwl.arvdocker.arv_docker_clear_cache()
 355
 356     def tearDown(self):
 357         root_logger = logging.getLogger('')
 358
 359         # Remove existing RuntimeStatusLoggingHandlers if they exist
 360         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 361         root_logger.handlers = handlers
 362
 363     @mock.patch("time.sleep")
 364     @stubs
 365     def test_submit_invalid_runner_ram(self, stubs, tm):
 366         exited = arvados_cwl.main(
 367             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 368              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 369             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 370         self.assertEqual(exited, 1)
 371
 372
 373     @stubs
 374     def test_submit_container(self, stubs):
 375         exited = arvados_cwl.main(
 376             ["--submit", "--no-wait", "--api=containers", "--debug",
 377                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 378             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 379
 380         stubs.api.collections().create.assert_has_calls([
 381             mock.call(body=JsonDiffMatcher({
 382                 'manifest_text':
 383                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 384                 'replication_desired': None,
 385                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 386             }), ensure_unique_name=False),
 387             mock.call(body=JsonDiffMatcher({
 388                 'manifest_text':
 389                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 390                 'replication_desired': None,
 391                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 392             }), ensure_unique_name=False),
 393             ])
 394
 395         expect_container = copy.deepcopy(stubs.expect_container_spec)
 396         stubs.api.container_requests().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_container))
 398         self.assertEqual(stubs.capture_stdout.getvalue(),
 399                          stubs.expect_container_request_uuid + '\n')
 400         self.assertEqual(exited, 0)
 401
 402
 403     @stubs
 404     def test_submit_container_tool(self, stubs):
 405         # test for issue #16139
 406         exited = arvados_cwl.main(
 407             ["--submit", "--no-wait", "--api=containers", "--debug",
 408                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 409             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 410
 411         self.assertEqual(stubs.capture_stdout.getvalue(),
 412                          stubs.expect_container_request_uuid + '\n')
 413         self.assertEqual(exited, 0)
 414
 415     @stubs
 416     def test_submit_container_no_reuse(self, stubs):
 417         exited = arvados_cwl.main(
 418             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 419                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 420             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 421
 422         expect_container = copy.deepcopy(stubs.expect_container_spec)
 423         expect_container["command"] = [
 424             'arvados-cwl-runner', '--local', '--api=containers',
 425             '--no-log-timestamps', '--disable-validate', '--disable-color',
 426             '--eval-timeout=20', '--thread-count=0',
 427             '--disable-reuse', "--collection-cache-size=256",
 428             '--debug', '--on-error=continue',
 429             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 430         expect_container["use_existing"] = False
 431
 432         stubs.api.container_requests().create.assert_called_with(
 433             body=JsonDiffMatcher(expect_container))
 434         self.assertEqual(stubs.capture_stdout.getvalue(),
 435                          stubs.expect_container_request_uuid + '\n')
 436         self.assertEqual(exited, 0)
 437
 438     @stubs
 439     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 440         exited = arvados_cwl.main(
 441             ["--submit", "--no-wait", "--api=containers", "--debug",
 442              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 443             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 444         self.assertEqual(exited, 0)
 445
 446         expect_container = copy.deepcopy(stubs.expect_container_spec)
 447         expect_container["command"] = [
 448             'arvados-cwl-runner', '--local', '--api=containers',
 449             '--no-log-timestamps', '--disable-validate', '--disable-color',
 450             '--eval-timeout=20', '--thread-count=0',
 451             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 452             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 453         expect_container["use_existing"] = False
 454         expect_container["name"] = "submit_wf_no_reuse.cwl"
 455         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 456             {
 457                 "class": "http://arvados.org/cwl#ReuseRequirement",
 458                 "enableReuse": False,
 459             },
 460         ]
 461         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 462             "arv": "http://arvados.org/cwl#",
 463             "cwltool": "http://commonwl.org/cwltool#"
 464         }
 465
 466         stubs.api.container_requests().create.assert_called_with(
 467             body=JsonDiffMatcher(expect_container))
 468         self.assertEqual(stubs.capture_stdout.getvalue(),
 469                          stubs.expect_container_request_uuid + '\n')
 470
 471
 472     @stubs
 473     def test_submit_container_on_error(self, stubs):
 474         exited = arvados_cwl.main(
 475             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 476                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 477             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 478
 479         expect_container = copy.deepcopy(stubs.expect_container_spec)
 480         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 481                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 482                                        '--eval-timeout=20', '--thread-count=0',
 483                                        '--enable-reuse', "--collection-cache-size=256",
 484                                        '--debug', '--on-error=stop',
 485                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 486
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=JsonDiffMatcher(expect_container))
 489         self.assertEqual(stubs.capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491         self.assertEqual(exited, 0)
 492
 493     @stubs
 494     def test_submit_container_output_name(self, stubs):
 495         output_name = "test_output_name"
 496
 497         exited = arvados_cwl.main(
 498             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 499                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 500             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 501
 502         expect_container = copy.deepcopy(stubs.expect_container_spec)
 503         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 504                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 505                                        '--eval-timeout=20', '--thread-count=0',
 506                                        '--enable-reuse', "--collection-cache-size=256",
 507                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 508                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 509         expect_container["output_name"] = output_name
 510
 511         stubs.api.container_requests().create.assert_called_with(
 512             body=JsonDiffMatcher(expect_container))
 513         self.assertEqual(stubs.capture_stdout.getvalue(),
 514                          stubs.expect_container_request_uuid + '\n')
 515         self.assertEqual(exited, 0)
 516
 517     @stubs
 518     def test_submit_storage_classes(self, stubs):
 519         exited = arvados_cwl.main(
 520             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 521                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 522             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 523
 524         expect_container = copy.deepcopy(stubs.expect_container_spec)
 525         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 526                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 527                                        '--eval-timeout=20', '--thread-count=0',
 528                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 529                                        "--storage-classes=foo", '--on-error=continue',
 530                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 531
 532         stubs.api.container_requests().create.assert_called_with(
 533             body=JsonDiffMatcher(expect_container))
 534         self.assertEqual(stubs.capture_stdout.getvalue(),
 535                          stubs.expect_container_request_uuid + '\n')
 536         self.assertEqual(exited, 0)
 537
 538     @stubs
 539     def test_submit_multiple_storage_classes(self, stubs):
 540         exited = arvados_cwl.main(
 541             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 542                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 543             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 544
 545         expect_container = copy.deepcopy(stubs.expect_container_spec)
 546         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 547                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 548                                        '--eval-timeout=20', '--thread-count=0',
 549                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 550                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 551                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 552
 553         stubs.api.container_requests().create.assert_called_with(
 554             body=JsonDiffMatcher(expect_container))
 555         self.assertEqual(stubs.capture_stdout.getvalue(),
 556                          stubs.expect_container_request_uuid + '\n')
 557         self.assertEqual(exited, 0)
 558
 559     @mock.patch("cwltool.task_queue.TaskQueue")
 560     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 561     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 562     @stubs
 563     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 564         final_output_c = arvados.collection.Collection()
 565         make_output.return_value = ({},final_output_c)
 566
 567         def set_final_output(job_order, output_callback, runtimeContext):
 568             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 569             return []
 570         job.side_effect = set_final_output
 571
 572         exited = arvados_cwl.main(
 573             ["--debug", "--local", "--storage-classes=foo",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 578         self.assertEqual(exited, 0)
 579
 580     @mock.patch("cwltool.task_queue.TaskQueue")
 581     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 582     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 583     @stubs
 584     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 585         final_output_c = arvados.collection.Collection()
 586         make_output.return_value = ({},final_output_c)
 587         stubs.api.config().get.return_value = {"default": {"Default": True}}
 588
 589         def set_final_output(job_order, output_callback, runtimeContext):
 590             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 591             return []
 592         job.side_effect = set_final_output
 593
 594         exited = arvados_cwl.main(
 595             ["--debug", "--local",
 596                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 597             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 598
 599         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 600         self.assertEqual(exited, 0)
 601
 602     @mock.patch("cwltool.task_queue.TaskQueue")
 603     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 604     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 605     @stubs
 606     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 607         final_output_c = arvados.collection.Collection()
 608         make_output.return_value = ({},final_output_c)
 609
 610         def set_final_output(job_order, output_callback, runtimeContext):
 611             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 612             return []
 613         job.side_effect = set_final_output
 614
 615         exited = arvados_cwl.main(
 616             ["--debug", "--local",
 617                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 618             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 619
 620         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 621         self.assertEqual(exited, 0)
 622
 623     @stubs
 624     def test_submit_container_output_ttl(self, stubs):
 625         exited = arvados_cwl.main(
 626             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 627                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 628             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 629
 630         expect_container = copy.deepcopy(stubs.expect_container_spec)
 631         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 632                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 633                                        '--eval-timeout=20', '--thread-count=0',
 634                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 635                                        '--on-error=continue',
 636                                        "--intermediate-output-ttl=3600",
 637                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 638
 639         stubs.api.container_requests().create.assert_called_with(
 640             body=JsonDiffMatcher(expect_container))
 641         self.assertEqual(stubs.capture_stdout.getvalue(),
 642                          stubs.expect_container_request_uuid + '\n')
 643         self.assertEqual(exited, 0)
 644
 645     @stubs
 646     def test_submit_container_trash_intermediate(self, stubs):
 647         exited = arvados_cwl.main(
 648             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 649                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 650             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 651
 652
 653         expect_container = copy.deepcopy(stubs.expect_container_spec)
 654         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 655                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 656                                        '--eval-timeout=20', '--thread-count=0',
 657                                        '--enable-reuse', "--collection-cache-size=256",
 658                                        '--debug', '--on-error=continue',
 659                                        "--trash-intermediate",
 660                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 661
 662         stubs.api.container_requests().create.assert_called_with(
 663             body=JsonDiffMatcher(expect_container))
 664         self.assertEqual(stubs.capture_stdout.getvalue(),
 665                          stubs.expect_container_request_uuid + '\n')
 666         self.assertEqual(exited, 0)
 667
 668     @stubs
 669     def test_submit_container_output_tags(self, stubs):
 670         output_tags = "tag0,tag1,tag2"
 671
 672         exited = arvados_cwl.main(
 673             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 674                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 676
 677         expect_container = copy.deepcopy(stubs.expect_container_spec)
 678         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 679                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 680                                        '--eval-timeout=20', '--thread-count=0',
 681                                        '--enable-reuse', "--collection-cache-size=256",
 682                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 683                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 684
 685         stubs.api.container_requests().create.assert_called_with(
 686             body=JsonDiffMatcher(expect_container))
 687         self.assertEqual(stubs.capture_stdout.getvalue(),
 688                          stubs.expect_container_request_uuid + '\n')
 689         self.assertEqual(exited, 0)
 690
 691     @stubs
 692     def test_submit_container_runner_ram(self, stubs):
 693         exited = arvados_cwl.main(
 694             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 695                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 696             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 697
 698         expect_container = copy.deepcopy(stubs.expect_container_spec)
 699         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 700
 701         stubs.api.container_requests().create.assert_called_with(
 702             body=JsonDiffMatcher(expect_container))
 703         self.assertEqual(stubs.capture_stdout.getvalue(),
 704                          stubs.expect_container_request_uuid + '\n')
 705         self.assertEqual(exited, 0)
 706
 707     @mock.patch("arvados.collection.CollectionReader")
 708     @mock.patch("time.sleep")
 709     @stubs
 710     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 711         collectionReader().exists.return_value = True
 712         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 713         exited = arvados_cwl.main(
 714             ["--submit", "--no-wait", "--api=containers", "--debug",
 715              "tests/wf/submit_keepref_wf.cwl"],
 716             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 717         self.assertEqual(exited, 0)
 718
 719     @mock.patch("arvados.collection.CollectionReader")
 720     @mock.patch("time.sleep")
 721     @stubs
 722     def test_submit_keepref(self, stubs, tm, reader):
 723         with open("tests/wf/expect_arvworkflow.cwl") as f:
 724             reader().open().__enter__().read.return_value = f.read()
 725
 726         exited = arvados_cwl.main(
 727             ["--submit", "--no-wait", "--api=containers", "--debug",
 728              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 729             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 730
 731         expect_container = {
 732             'priority': 500,
 733             'mounts': {
 734                 '/var/spool/cwl': {
 735                     'writable': True,
 736                     'kind': 'collection'
 737                 },
 738                 'stdout': {
 739                     'path': '/var/spool/cwl/cwl.output.json',
 740                     'kind': 'file'
 741                 },
 742                 '/var/lib/cwl/workflow': {
 743                     'portable_data_hash': '99999999999999999999999999999994+99',
 744                     'kind': 'collection'
 745                 },
 746                 '/var/lib/cwl/cwl.input.json': {
 747                     'content': {
 748                         'x': 'XxX'
 749                     },
 750                     'kind': 'json'
 751                 }
 752             }, 'state': 'Committed',
 753             'output_path': '/var/spool/cwl',
 754             'name': 'expect_arvworkflow.cwl#main',
 755             'container_image': '999999999999999999999999999999d3+99',
 756             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 757                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 758                         '--eval-timeout=20', '--thread-count=0',
 759                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 760                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 761             'cwd': '/var/spool/cwl',
 762             'runtime_constraints': {
 763                 'API': True,
 764                 'vcpus': 1,
 765                 'ram': 1342177280
 766             },
 767             'use_existing': False,
 768             'properties': {},
 769             'secret_mounts': {}
 770         }
 771
 772         stubs.api.container_requests().create.assert_called_with(
 773             body=JsonDiffMatcher(expect_container))
 774         self.assertEqual(stubs.capture_stdout.getvalue(),
 775                          stubs.expect_container_request_uuid + '\n')
 776         self.assertEqual(exited, 0)
 777
 778     @mock.patch("time.sleep")
 779     @stubs
 780     def test_submit_arvworkflow(self, stubs, tm):
 781         with open("tests/wf/expect_arvworkflow.cwl") as f:
 782             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 783
 784         exited = arvados_cwl.main(
 785             ["--submit", "--no-wait", "--api=containers", "--debug",
 786              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 787             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 788
 789         expect_container = {
 790             'priority': 500,
 791             'mounts': {
 792                 '/var/spool/cwl': {
 793                     'writable': True,
 794                     'kind': 'collection'
 795                 },
 796                 'stdout': {
 797                     'path': '/var/spool/cwl/cwl.output.json',
 798                     'kind': 'file'
 799                 },
 800                 '/var/lib/cwl/workflow.json': {
 801                     'kind': 'json',
 802                     'content': {
 803                         'cwlVersion': 'v1.0',
 804                         '$graph': [
 805                             {
 806                                 'id': '#main',
 807                                 'inputs': [
 808                                     {'type': 'string', 'id': '#main/x'}
 809                                 ],
 810                                 'steps': [
 811                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 812                                      'run': '#submit_tool.cwl',
 813                                      'id': '#main/step1',
 814                                      'out': []}
 815                                 ],
 816                                 'class': 'Workflow',
 817                                 'outputs': []
 818                             },
 819                             {
 820                                 'inputs': [
 821                                     {
 822                                         'inputBinding': {'position': 1},
 823                                         'type': 'string',
 824                                         'id': '#submit_tool.cwl/x'}
 825                                 ],
 826                                 'requirements': [
 827                                     {
 828                                         'dockerPull': 'debian:buster-slim',
 829                                         'class': 'DockerRequirement',
 830                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 831                                     }
 832                                 ],
 833                                 'id': '#submit_tool.cwl',
 834                                 'outputs': [],
 835                                 'baseCommand': 'cat',
 836                                 'class': 'CommandLineTool'
 837                             }
 838                         ]
 839                     }
 840                 },
 841                 '/var/lib/cwl/cwl.input.json': {
 842                     'content': {
 843                         'x': 'XxX'
 844                     },
 845                     'kind': 'json'
 846                 }
 847             }, 'state': 'Committed',
 848             'output_path': '/var/spool/cwl',
 849             'name': 'a test workflow',
 850             'container_image': "999999999999999999999999999999d3+99",
 851             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 852                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 853                         '--eval-timeout=20', '--thread-count=0',
 854                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 855                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 856             'cwd': '/var/spool/cwl',
 857             'runtime_constraints': {
 858                 'API': True,
 859                 'vcpus': 1,
 860                 'ram': 1342177280
 861             },
 862             'use_existing': False,
 863             'properties': {
 864                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 865             },
 866             'secret_mounts': {}
 867         }
 868
 869         stubs.api.container_requests().create.assert_called_with(
 870             body=JsonDiffMatcher(expect_container))
 871         self.assertEqual(stubs.capture_stdout.getvalue(),
 872                          stubs.expect_container_request_uuid + '\n')
 873         self.assertEqual(exited, 0)
 874
 875     @stubs
 876     def test_submit_container_name(self, stubs):
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 879                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["name"] = "hello container 123"
 884
 885         stubs.api.container_requests().create.assert_called_with(
 886             body=JsonDiffMatcher(expect_container))
 887         self.assertEqual(stubs.capture_stdout.getvalue(),
 888                          stubs.expect_container_request_uuid + '\n')
 889         self.assertEqual(exited, 0)
 890
 891     @stubs
 892     def test_submit_missing_input(self, stubs):
 893         exited = arvados_cwl.main(
 894             ["--submit", "--no-wait", "--api=containers", "--debug",
 895              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 896             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 897         self.assertEqual(exited, 0)
 898
 899         exited = arvados_cwl.main(
 900             ["--submit", "--no-wait", "--api=containers", "--debug",
 901              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 902             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 903         self.assertEqual(exited, 1)
 904
 905     @stubs
 906     def test_submit_container_project(self, stubs):
 907         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 908         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 909         exited = arvados_cwl.main(
 910             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 911                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 912             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 913
 914         expect_container = copy.deepcopy(stubs.expect_container_spec)
 915         expect_container["owner_uuid"] = project_uuid
 916         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 917                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 918                                        "--eval-timeout=20", "--thread-count=0",
 919                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 920                                        '--on-error=continue',
 921                                        '--project-uuid='+project_uuid,
 922                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 923
 924         stubs.api.container_requests().create.assert_called_with(
 925             body=JsonDiffMatcher(expect_container))
 926         self.assertEqual(stubs.capture_stdout.getvalue(),
 927                          stubs.expect_container_request_uuid + '\n')
 928         self.assertEqual(exited, 0)
 929
 930     @stubs
 931     def test_submit_container_eval_timeout(self, stubs):
 932         exited = arvados_cwl.main(
 933             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 934                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 935             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 936
 937         expect_container = copy.deepcopy(stubs.expect_container_spec)
 938         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 939                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 940                                        '--eval-timeout=60.0', '--thread-count=0',
 941                                        '--enable-reuse', "--collection-cache-size=256",
 942                                        '--debug', '--on-error=continue',
 943                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 944
 945         stubs.api.container_requests().create.assert_called_with(
 946             body=JsonDiffMatcher(expect_container))
 947         self.assertEqual(stubs.capture_stdout.getvalue(),
 948                          stubs.expect_container_request_uuid + '\n')
 949         self.assertEqual(exited, 0)
 950
 951     @stubs
 952     def test_submit_container_collection_cache(self, stubs):
 953         exited = arvados_cwl.main(
 954             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 955                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 956             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 957
 958         expect_container = copy.deepcopy(stubs.expect_container_spec)
 959         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 960                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 961                                        '--eval-timeout=20', '--thread-count=0',
 962                                        '--enable-reuse', "--collection-cache-size=500",
 963                                        '--debug', '--on-error=continue',
 964                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 965         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 966
 967         stubs.api.container_requests().create.assert_called_with(
 968             body=JsonDiffMatcher(expect_container))
 969         self.assertEqual(stubs.capture_stdout.getvalue(),
 970                          stubs.expect_container_request_uuid + '\n')
 971         self.assertEqual(exited, 0)
 972
 973     @stubs
 974     def test_submit_container_thread_count(self, stubs):
 975         exited = arvados_cwl.main(
 976             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 977                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 978             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 979
 980         expect_container = copy.deepcopy(stubs.expect_container_spec)
 981         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 982                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 983                                        '--eval-timeout=20', '--thread-count=20',
 984                                        '--enable-reuse', "--collection-cache-size=256",
 985                                        '--debug', '--on-error=continue',
 986                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 987
 988         stubs.api.container_requests().create.assert_called_with(
 989             body=JsonDiffMatcher(expect_container))
 990         self.assertEqual(stubs.capture_stdout.getvalue(),
 991                          stubs.expect_container_request_uuid + '\n')
 992         self.assertEqual(exited, 0)
 993
 994     @stubs
 995     def test_submit_container_runner_image(self, stubs):
 996         exited = arvados_cwl.main(
 997             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 998                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 999             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1000
1001         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1002
1003         expect_container = copy.deepcopy(stubs.expect_container_spec)
1004         stubs.api.container_requests().create.assert_called_with(
1005             body=JsonDiffMatcher(expect_container))
1006         self.assertEqual(stubs.capture_stdout.getvalue(),
1007                          stubs.expect_container_request_uuid + '\n')
1008         self.assertEqual(exited, 0)
1009
1010     @stubs
1011     def test_submit_priority(self, stubs):
1012         exited = arvados_cwl.main(
1013             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1014                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1015             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1016
1017         stubs.expect_container_spec["priority"] = 669
1018
1019         expect_container = copy.deepcopy(stubs.expect_container_spec)
1020         stubs.api.container_requests().create.assert_called_with(
1021             body=JsonDiffMatcher(expect_container))
1022         self.assertEqual(stubs.capture_stdout.getvalue(),
1023                          stubs.expect_container_request_uuid + '\n')
1024         self.assertEqual(exited, 0)
1025
1026     @stubs
1027     def test_submit_wf_runner_resources(self, stubs):
1028         exited = arvados_cwl.main(
1029             ["--submit", "--no-wait", "--api=containers", "--debug",
1030                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1031             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1032
1033         expect_container = copy.deepcopy(stubs.expect_container_spec)
1034         expect_container["runtime_constraints"] = {
1035             "API": True,
1036             "vcpus": 2,
1037             "ram": (2000+512) * 2**20
1038         }
1039         expect_container["name"] = "submit_wf_runner_resources.cwl"
1040         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1041             {
1042                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1043                 "coresMin": 2,
1044                 "ramMin": 2000,
1045                 "keep_cache": 512
1046             }
1047         ]
1048         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1049             "arv": "http://arvados.org/cwl#",
1050         }
1051         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1052                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1053                         '--eval-timeout=20', '--thread-count=0',
1054                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1055                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1056
1057         stubs.api.container_requests().create.assert_called_with(
1058             body=JsonDiffMatcher(expect_container))
1059         self.assertEqual(stubs.capture_stdout.getvalue(),
1060                          stubs.expect_container_request_uuid + '\n')
1061         self.assertEqual(exited, 0)
1062
1063     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1064     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1065     @mock.patch("arvados.api")
1066     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1067         arvados_cwl.arvdocker.arv_docker_clear_cache()
1068
1069         arvrunner = mock.MagicMock()
1070         arvrunner.project_uuid = ""
1071         api.return_value = mock.MagicMock()
1072         arvrunner.api = api.return_value
1073         arvrunner.runtimeContext.match_local_docker = False
1074         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1075                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1076                                                                         "link_class": "docker_image_repo+tag",
1077                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1078                                                                         "owner_uuid": "",
1079                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1080                                                             {"items": [{"created_at": "",
1081                                                                         "head_uuid": "",
1082                                                                         "link_class": "docker_image_hash",
1083                                                                         "name": "123456",
1084                                                                         "owner_uuid": "",
1085                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1086         )
1087         find_one_image_hash.return_value = "123456"
1088
1089         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1090                                                                               "owner_uuid": "",
1091                                                                               "manifest_text": "",
1092                                                                               "properties": ""
1093                                                                           }], "items_available": 1, "offset": 0},)
1094         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1095         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1096                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1097         self.assertEqual("9999999999999999999999999999999b+99",
1098                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1099
1100
1101     @stubs
1102     def test_submit_secrets(self, stubs):
1103         exited = arvados_cwl.main(
1104             ["--submit", "--no-wait", "--api=containers", "--debug",
1105                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1106             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1107
1108         expect_container = {
1109             "command": [
1110                 "arvados-cwl-runner",
1111                 "--local",
1112                 "--api=containers",
1113                 "--no-log-timestamps",
1114                 "--disable-validate",
1115                 "--disable-color",
1116                 "--eval-timeout=20",
1117                 '--thread-count=0',
1118                 "--enable-reuse",
1119                 "--collection-cache-size=256",
1120                 '--debug',
1121                 "--on-error=continue",
1122                 "/var/lib/cwl/workflow.json#main",
1123                 "/var/lib/cwl/cwl.input.json"
1124             ],
1125             "container_image": "999999999999999999999999999999d3+99",
1126             "cwd": "/var/spool/cwl",
1127             "mounts": {
1128                 "/var/lib/cwl/cwl.input.json": {
1129                     "content": {
1130                         "pw": {
1131                             "$include": "/secrets/s0"
1132                         }
1133                     },
1134                     "kind": "json"
1135                 },
1136                 "/var/lib/cwl/workflow.json": {
1137                     "content": {
1138                         "$graph": [
1139                             {
1140                                 "arguments": [
1141                                     "md5sum",
1142                                     "example.conf"
1143                                 ],
1144                                 "class": "CommandLineTool",
1145                                 "hints": [
1146                                     {
1147                                         "class": "http://commonwl.org/cwltool#Secrets",
1148                                         "secrets": [
1149                                             "#secret_job.cwl/pw"
1150                                         ]
1151                                     }
1152                                 ],
1153                                 "id": "#secret_job.cwl",
1154                                 "inputs": [
1155                                     {
1156                                         "id": "#secret_job.cwl/pw",
1157                                         "type": "string"
1158                                     }
1159                                 ],
1160                                 "outputs": [
1161                                     {
1162                                         "id": "#secret_job.cwl/out",
1163                                         "type": "File",
1164                                         "outputBinding": {
1165                                               "glob": "hashed_example.txt"
1166                                         }
1167                                     }
1168                                 ],
1169                                 "stdout": "hashed_example.txt",
1170                                 "requirements": [
1171                                     {
1172                                         "class": "InitialWorkDirRequirement",
1173                                         "listing": [
1174                                             {
1175                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1176                                                 "entryname": "example.conf"
1177                                             }
1178                                         ]
1179                                     }
1180                                 ]
1181                             },
1182                             {
1183                                 "class": "Workflow",
1184                                 "hints": [
1185                                     {
1186                                         "class": "DockerRequirement",
1187                                         "dockerPull": "debian:buster-slim",
1188                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1189                                     },
1190                                     {
1191                                         "class": "http://commonwl.org/cwltool#Secrets",
1192                                         "secrets": [
1193                                             "#main/pw"
1194                                         ]
1195                                     }
1196                                 ],
1197                                 "id": "#main",
1198                                 "inputs": [
1199                                     {
1200                                         "id": "#main/pw",
1201                                         "type": "string"
1202                                     }
1203                                 ],
1204                                 "outputs": [
1205                                     {
1206                                         "id": "#main/out",
1207                                         "outputSource": "#main/step1/out",
1208                                         "type": "File"
1209                                     }
1210                                 ],
1211                                 "steps": [
1212                                     {
1213                                         "id": "#main/step1",
1214                                         "in": [
1215                                             {
1216                                                 "id": "#main/step1/pw",
1217                                                 "source": "#main/pw"
1218                                             }
1219                                         ],
1220                                         "out": [
1221                                             "#main/step1/out"
1222                                         ],
1223                                         "run": "#secret_job.cwl"
1224                                     }
1225                                 ]
1226                             }
1227                         ],
1228                         "$namespaces": {
1229                             "cwltool": "http://commonwl.org/cwltool#"
1230                         },
1231                         "cwlVersion": "v1.0"
1232                     },
1233                     "kind": "json"
1234                 },
1235                 "/var/spool/cwl": {
1236                     "kind": "collection",
1237                     "writable": True
1238                 },
1239                 "stdout": {
1240                     "kind": "file",
1241                     "path": "/var/spool/cwl/cwl.output.json"
1242                 }
1243             },
1244             "name": "secret_wf.cwl",
1245             "output_path": "/var/spool/cwl",
1246             "priority": 500,
1247             "properties": {},
1248             "runtime_constraints": {
1249                 "API": True,
1250                 "ram": 1342177280,
1251                 "vcpus": 1
1252             },
1253             "secret_mounts": {
1254                 "/secrets/s0": {
1255                     "content": "blorp",
1256                     "kind": "text"
1257                 }
1258             },
1259             "state": "Committed",
1260             "use_existing": False
1261         }
1262
1263         stubs.api.container_requests().create.assert_called_with(
1264             body=JsonDiffMatcher(expect_container))
1265         self.assertEqual(stubs.capture_stdout.getvalue(),
1266                          stubs.expect_container_request_uuid + '\n')
1267         self.assertEqual(exited, 0)
1268
1269     @stubs
1270     def test_submit_request_uuid(self, stubs):
1271         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1272         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1273
1274         stubs.api.container_requests().update().execute.return_value = {
1275             "uuid": stubs.expect_container_request_uuid,
1276             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1277             "state": "Queued"
1278         }
1279
1280         exited = arvados_cwl.main(
1281             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1282                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1283             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1284
1285         stubs.api.container_requests().update.assert_called_with(
1286             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1287         self.assertEqual(stubs.capture_stdout.getvalue(),
1288                          stubs.expect_container_request_uuid + '\n')
1289         self.assertEqual(exited, 0)
1290
1291     @stubs
1292     def test_submit_container_cluster_id(self, stubs):
1293         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1294
1295         exited = arvados_cwl.main(
1296             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1297                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1298             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1299
1300         expect_container = copy.deepcopy(stubs.expect_container_spec)
1301
1302         stubs.api.container_requests().create.assert_called_with(
1303             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1304         self.assertEqual(stubs.capture_stdout.getvalue(),
1305                          stubs.expect_container_request_uuid + '\n')
1306         self.assertEqual(exited, 0)
1307
1308     @stubs
1309     def test_submit_validate_cluster_id(self, stubs):
1310         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1311         exited = arvados_cwl.main(
1312             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1313              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1314             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1315         self.assertEqual(exited, 1)
1316
1317     @stubs
1318     def test_submit_validate_project_uuid(self, stubs):
1319         # Fails with bad cluster prefix
1320         exited = arvados_cwl.main(
1321             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1322              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1323             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1324         self.assertEqual(exited, 1)
1325
1326         # Project lookup fails
1327         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1328         exited = arvados_cwl.main(
1329             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1330              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1331             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1332         self.assertEqual(exited, 1)
1333
1334         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1335         exited = arvados_cwl.main(
1336             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1337              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1338             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1339         self.assertEqual(exited, 0)
1340
1341
1342     @mock.patch("arvados.collection.CollectionReader")
1343     @stubs
1344     def test_submit_uuid_inputs(self, stubs, collectionReader):
1345         collectionReader().exists.return_value = True
1346         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1347         def list_side_effect(**kwargs):
1348             m = mock.MagicMock()
1349             if "count" in kwargs:
1350                 m.execute.return_value = {"items": [
1351                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1352                 ]}
1353             else:
1354                 m.execute.return_value = {"items": []}
1355             return m
1356         stubs.api.collections().list.side_effect = list_side_effect
1357
1358         exited = arvados_cwl.main(
1359             ["--submit", "--no-wait", "--api=containers", "--debug",
1360                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1361             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1362
1363         expect_container = copy.deepcopy(stubs.expect_container_spec)
1364         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1365         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1366         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1367
1368         stubs.api.collections().list.assert_has_calls([
1369             mock.call(count='none',
1370                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1371                       select=['uuid', 'portable_data_hash'])])
1372         stubs.api.container_requests().create.assert_called_with(
1373             body=JsonDiffMatcher(expect_container))
1374         self.assertEqual(stubs.capture_stdout.getvalue(),
1375                          stubs.expect_container_request_uuid + '\n')
1376         self.assertEqual(exited, 0)
1377
1378     @stubs
1379     def test_submit_mismatched_uuid_inputs(self, stubs):
1380         def list_side_effect(**kwargs):
1381             m = mock.MagicMock()
1382             if "count" in kwargs:
1383                 m.execute.return_value = {"items": [
1384                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1385                 ]}
1386             else:
1387                 m.execute.return_value = {"items": []}
1388             return m
1389         stubs.api.collections().list.side_effect = list_side_effect
1390
1391         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1392             capture_stderr = StringIO()
1393             cwltool_logger = logging.getLogger('cwltool')
1394             stderr_logger = logging.StreamHandler(capture_stderr)
1395             cwltool_logger.addHandler(stderr_logger)
1396
1397             try:
1398                 exited = arvados_cwl.main(
1399                     ["--submit", "--no-wait", "--api=containers", "--debug",
1400                         "tests/wf/submit_wf.cwl", infile],
1401                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1402
1403                 self.assertEqual(exited, 1)
1404                 self.assertRegex(
1405                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1406                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1407             finally:
1408                 cwltool_logger.removeHandler(stderr_logger)
1409
1410     @mock.patch("arvados.collection.CollectionReader")
1411     @stubs
1412     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1413         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1414         capture_stderr = StringIO()
1415
1416         cwltool_logger = logging.getLogger('cwltool')
1417         stderr_logger = logging.StreamHandler(capture_stderr)
1418         cwltool_logger.addHandler(stderr_logger)
1419
1420         exited = arvados_cwl.main(
1421             ["--submit", "--no-wait", "--api=containers", "--debug",
1422                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1423             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1424
1425         try:
1426             self.assertEqual(exited, 1)
1427             self.assertRegex(
1428                 capture_stderr.getvalue(),
1429                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1430         finally:
1431             cwltool_logger.removeHandler(stderr_logger)
1432
1433     @stubs
1434     def test_submit_set_process_properties(self, stubs):
1435         exited = arvados_cwl.main(
1436             ["--submit", "--no-wait", "--api=containers", "--debug",
1437                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1438             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1439
1440         expect_container = copy.deepcopy(stubs.expect_container_spec)
1441         expect_container["name"] = "submit_wf_process_properties.cwl"
1442         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1443             {
1444                 "class": "http://arvados.org/cwl#ProcessProperties",
1445                 "processProperties": [
1446                     {"propertyName": "baz",
1447                      "propertyValue": "$(inputs.x.basename)"},
1448                     {"propertyName": "foo",
1449                      "propertyValue": "bar"},
1450                     {"propertyName": "quux",
1451                      "propertyValue": {
1452                          "q1": 1,
1453                          "q2": 2
1454                      }
1455                     }
1456                 ],
1457             }
1458         ]
1459         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1460             "arv": "http://arvados.org/cwl#"
1461         }
1462
1463         expect_container["properties"] = {
1464             "baz": "blorp.txt",
1465             "foo": "bar",
1466             "quux": {
1467                 "q1": 1,
1468                 "q2": 2
1469             }
1470         }
1471
1472         stubs.api.container_requests().create.assert_called_with(
1473             body=JsonDiffMatcher(expect_container))
1474         self.assertEqual(stubs.capture_stdout.getvalue(),
1475                          stubs.expect_container_request_uuid + '\n')
1476         self.assertEqual(exited, 0)
1477
1478
1479     @stubs
1480     def test_submit_enable_preemptible(self, stubs):
1481         exited = arvados_cwl.main(
1482             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1483                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1484             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1485
1486         expect_container = copy.deepcopy(stubs.expect_container_spec)
1487         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1488                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1489                         '--eval-timeout=20', '--thread-count=0',
1490                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1491                                        '--enable-preemptible',
1492                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1493
1494         stubs.api.container_requests().create.assert_called_with(
1495             body=JsonDiffMatcher(expect_container))
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_container_request_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500     @stubs
1501     def test_submit_disable_preemptible(self, stubs):
1502         exited = arvados_cwl.main(
1503             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1504                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1505             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1506
1507         expect_container = copy.deepcopy(stubs.expect_container_spec)
1508         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1509                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1510                         '--eval-timeout=20', '--thread-count=0',
1511                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1512                                        '--disable-preemptible',
1513                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1514
1515         stubs.api.container_requests().create.assert_called_with(
1516             body=JsonDiffMatcher(expect_container))
1517         self.assertEqual(stubs.capture_stdout.getvalue(),
1518                          stubs.expect_container_request_uuid + '\n')
1519         self.assertEqual(exited, 0)
1520
1521
1522 class TestCreateWorkflow(unittest.TestCase):
1523     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1524     expect_workflow = StripYAMLComments(
1525         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1526
1527     def setUp(self):
1528         cwltool.process._names = set()
1529         arvados_cwl.arvdocker.arv_docker_clear_cache()
1530
1531     def tearDown(self):
1532         root_logger = logging.getLogger('')
1533
1534         # Remove existing RuntimeStatusLoggingHandlers if they exist
1535         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1536         root_logger.handlers = handlers
1537
1538     @stubs
1539     def test_create(self, stubs):
1540         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1541         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1542
1543         exited = arvados_cwl.main(
1544             ["--create-workflow", "--debug",
1545              "--api=containers",
1546              "--project-uuid", project_uuid,
1547              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1548             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1549
1550         stubs.api.pipeline_templates().create.refute_called()
1551         stubs.api.container_requests().create.refute_called()
1552
1553         body = {
1554             "workflow": {
1555                 "owner_uuid": project_uuid,
1556                 "name": "submit_wf.cwl",
1557                 "description": "",
1558                 "definition": self.expect_workflow,
1559             }
1560         }
1561         stubs.api.workflows().create.assert_called_with(
1562             body=JsonDiffMatcher(body))
1563
1564         self.assertEqual(stubs.capture_stdout.getvalue(),
1565                          stubs.expect_workflow_uuid + '\n')
1566         self.assertEqual(exited, 0)
1567
1568     @stubs
1569     def test_create_name(self, stubs):
1570         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1571         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1572
1573         exited = arvados_cwl.main(
1574             ["--create-workflow", "--debug",
1575              "--api=containers",
1576              "--project-uuid", project_uuid,
1577              "--name", "testing 123",
1578              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1579             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1580
1581         stubs.api.pipeline_templates().create.refute_called()
1582         stubs.api.container_requests().create.refute_called()
1583
1584         body = {
1585             "workflow": {
1586                 "owner_uuid": project_uuid,
1587                 "name": "testing 123",
1588                 "description": "",
1589                 "definition": self.expect_workflow,
1590             }
1591         }
1592         stubs.api.workflows().create.assert_called_with(
1593             body=JsonDiffMatcher(body))
1594
1595         self.assertEqual(stubs.capture_stdout.getvalue(),
1596                          stubs.expect_workflow_uuid + '\n')
1597         self.assertEqual(exited, 0)
1598
1599
1600     @stubs
1601     def test_update(self, stubs):
1602         exited = arvados_cwl.main(
1603             ["--update-workflow", self.existing_workflow_uuid,
1604              "--debug",
1605              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1606             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1607
1608         body = {
1609             "workflow": {
1610                 "name": "submit_wf.cwl",
1611                 "description": "",
1612                 "definition": self.expect_workflow,
1613             }
1614         }
1615         stubs.api.workflows().update.assert_called_with(
1616             uuid=self.existing_workflow_uuid,
1617             body=JsonDiffMatcher(body))
1618         self.assertEqual(stubs.capture_stdout.getvalue(),
1619                          self.existing_workflow_uuid + '\n')
1620         self.assertEqual(exited, 0)
1621
1622
1623     @stubs
1624     def test_update_name(self, stubs):
1625         exited = arvados_cwl.main(
1626             ["--update-workflow", self.existing_workflow_uuid,
1627              "--debug", "--name", "testing 123",
1628              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1629             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1630
1631         body = {
1632             "workflow": {
1633                 "name": "testing 123",
1634                 "description": "",
1635                 "definition": self.expect_workflow,
1636             }
1637         }
1638         stubs.api.workflows().update.assert_called_with(
1639             uuid=self.existing_workflow_uuid,
1640             body=JsonDiffMatcher(body))
1641         self.assertEqual(stubs.capture_stdout.getvalue(),
1642                          self.existing_workflow_uuid + '\n')
1643         self.assertEqual(exited, 0)
1644
1645     @stubs
1646     def test_create_collection_per_tool(self, stubs):
1647         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1648         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1649
1650         exited = arvados_cwl.main(
1651             ["--create-workflow", "--debug",
1652              "--api=containers",
1653              "--project-uuid", project_uuid,
1654              "tests/collection_per_tool/collection_per_tool.cwl"],
1655             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1656
1657         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1658         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1659
1660         body = {
1661             "workflow": {
1662                 "owner_uuid": project_uuid,
1663                 "name": "collection_per_tool.cwl",
1664                 "description": "",
1665                 "definition": expect_workflow,
1666             }
1667         }
1668         stubs.api.workflows().create.assert_called_with(
1669             body=JsonDiffMatcher(body))
1670
1671         self.assertEqual(stubs.capture_stdout.getvalue(),
1672                          stubs.expect_workflow_uuid + '\n')
1673         self.assertEqual(exited, 0)
1674
1675     @stubs
1676     def test_create_with_imports(self, stubs):
1677         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1678         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1679
1680         exited = arvados_cwl.main(
1681             ["--create-workflow", "--debug",
1682              "--api=containers",
1683              "--project-uuid", project_uuid,
1684              "tests/wf/feddemo/feddemo.cwl"],
1685             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1686
1687         stubs.api.pipeline_templates().create.refute_called()
1688         stubs.api.container_requests().create.refute_called()
1689
1690         self.assertEqual(stubs.capture_stdout.getvalue(),
1691                          stubs.expect_workflow_uuid + '\n')
1692         self.assertEqual(exited, 0)
1693
1694     @stubs
1695     def test_create_with_no_input(self, stubs):
1696         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1697         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1698
1699         exited = arvados_cwl.main(
1700             ["--create-workflow", "--debug",
1701              "--api=containers",
1702              "--project-uuid", project_uuid,
1703              "tests/wf/revsort/revsort.cwl"],
1704             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1705
1706         stubs.api.pipeline_templates().create.refute_called()
1707         stubs.api.container_requests().create.refute_called()
1708
1709         self.assertEqual(stubs.capture_stdout.getvalue(),
1710                          stubs.expect_workflow_uuid + '\n')
1711         self.assertEqual(exited, 0)