sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22
  23 from io import BytesIO
  24
  25 # StringIO.StringIO and io.StringIO have different behavior write() is
  26 # called with both python2 (byte) strings and unicode strings
  27 # (specifically there's some logging in cwltool that causes trouble).
  28 # This isn't a problem on python3 because all string are unicode.
  29 if sys.version_info[0] < 3:
  30     from StringIO import StringIO
  31 else:
  32     from io import StringIO
  33
  34 import arvados
  35 import arvados.collection
  36 import arvados_cwl
  37 import arvados_cwl.executor
  38 import arvados_cwl.runner
  39 import arvados.keep
  40
  41 from .matcher import JsonDiffMatcher, StripYAMLComments
  42 from .mock_discovery import get_rootDesc
  43
  44 import ruamel.yaml as yaml
  45
  46 _rootDesc = None
  47
  48 def stubs(func):
  49     @functools.wraps(func)
  50     @mock.patch("uuid.uuid4")
  51     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  52     @mock.patch("arvados.collection.KeepClient")
  53     @mock.patch("arvados.keep.KeepClient")
  54     @mock.patch("arvados.events.subscribe")
  55     def wrapped(self, events, keep_client1, keep_client2, keepdocker, uuid4, *args, **kwargs):
  56         class Stubs(object):
  57             pass
  58         stubs = Stubs()
  59         stubs.events = events
  60         stubs.keepdocker = keepdocker
  61
  62         uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  63                              "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  64                              "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  65
  66         def putstub(p, **kwargs):
  67             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  68         keep_client1().put.side_effect = putstub
  69         keep_client1.put.side_effect = putstub
  70         keep_client2().put.side_effect = putstub
  71         keep_client2.put.side_effect = putstub
  72
  73         stubs.keep_client = keep_client2
  74         stubs.docker_images = {
  75             "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  76             "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  77             "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  78             "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  79         }
  80         def kd(a, b, image_name=None, image_tag=None):
  81             return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  82         stubs.keepdocker.side_effect = kd
  83
  84         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  85         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  86
  87         if sys.version_info[0] < 3:
  88             stubs.capture_stdout = BytesIO()
  89         else:
  90             stubs.capture_stdout = StringIO()
  91
  92         stubs.api = mock.MagicMock()
  93         stubs.api._rootDesc = get_rootDesc()
  94         stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  95         stubs.api._rootDesc["revision"] = "20210628"
  96
  97         stubs.api.users().current().execute.return_value = {
  98             "uuid": stubs.fake_user_uuid,
  99         }
 100         stubs.api.collections().list().execute.return_value = {"items": []}
 101         stubs.api.containers().current().execute.return_value = {
 102             "uuid": stubs.fake_container_uuid,
 103         }
 104         stubs.api.config()["StorageClasses"].items.return_value = {
 105             "default": {
 106                 "Default": True
 107             }
 108         }.items()
 109
 110         class CollectionExecute(object):
 111             def __init__(self, exe):
 112                 self.exe = exe
 113             def execute(self, num_retries=None):
 114                 return self.exe
 115
 116         def collection_createstub(created_collections, body, ensure_unique_name=None):
 117             mt = body["manifest_text"].encode('utf-8')
 118             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 119             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 120             created_collections[uuid] = {
 121                 "uuid": uuid,
 122                 "portable_data_hash": pdh,
 123                 "manifest_text": mt.decode('utf-8')
 124             }
 125             return CollectionExecute(created_collections[uuid])
 126
 127         def collection_getstub(created_collections, uuid):
 128             for v in viewvalues(created_collections):
 129                 if uuid in (v["uuid"], v["portable_data_hash"]):
 130                     return CollectionExecute(v)
 131
 132         created_collections = {
 133             "99999999999999999999999999999998+99": {
 134                 "uuid": "",
 135                 "portable_data_hash": "99999999999999999999999999999998+99",
 136                 "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 137             },
 138             "99999999999999999999999999999997+99": {
 139                 "uuid": "",
 140                 "portable_data_hash": "99999999999999999999999999999997+99",
 141                 "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 142             },
 143             "99999999999999999999999999999994+99": {
 144                 "uuid": "",
 145                 "portable_data_hash": "99999999999999999999999999999994+99",
 146                 "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 147             },
 148             "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 149                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 150                 "portable_data_hash": "999999999999999999999999999999d3+99",
 151                 "manifest_text": ""
 152             },
 153             "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 154                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 155                 "portable_data_hash": "999999999999999999999999999999d4+99",
 156                 "manifest_text": ""
 157             },
 158             "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 159                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 160                 "portable_data_hash": "999999999999999999999999999999d5+99",
 161                 "manifest_text": ""
 162             },
 163             "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 164                 "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 165                 "portable_data_hash": "999999999999999999999999999999d6+99",
 166                 "manifest_text": ""
 167             }
 168         }
 169         stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 170         stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 171
 172         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 173         stubs.api.jobs().create().execute.return_value = {
 174             "uuid": stubs.expect_job_uuid,
 175             "state": "Queued",
 176         }
 177
 178         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 179         stubs.api.container_requests().create().execute.return_value = {
 180             "uuid": stubs.expect_container_request_uuid,
 181             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 182             "state": "Queued"
 183         }
 184
 185         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 186         stubs.api.pipeline_templates().create().execute.return_value = {
 187             "uuid": stubs.expect_pipeline_template_uuid,
 188         }
 189         stubs.expect_job_spec = {
 190             'runtime_constraints': {
 191                 'docker_image': '999999999999999999999999999999d3+99',
 192                 'min_ram_mb_per_node': 1024
 193             },
 194             'script_parameters': {
 195                 'x': {
 196                     'basename': 'blorp.txt',
 197                     'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 198                     'class': 'File'
 199                 },
 200                 'y': {
 201                     'basename': '99999999999999999999999999999998+99',
 202                     'location': 'keep:99999999999999999999999999999998+99',
 203                     'class': 'Directory'
 204                 },
 205                 'z': {
 206                     'basename': 'anonymous',
 207                     "listing": [{
 208                         "basename": "renamed.txt",
 209                         "class": "File",
 210                         "location": "keep:99999999999999999999999999999998+99/file1.txt",
 211                         "size": 0
 212                     }],
 213                     'class': 'Directory'
 214                 },
 215                 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 216             },
 217             'repository': 'arvados',
 218             'script_version': 'master',
 219             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 220             'script': 'cwl-runner'
 221         }
 222         stubs.pipeline_component = stubs.expect_job_spec.copy()
 223         stubs.expect_pipeline_instance = {
 224             'name': 'submit_wf.cwl',
 225             'state': 'RunningOnServer',
 226             'owner_uuid': None,
 227             "components": {
 228                 "cwl-runner": {
 229                     'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 230                     'script_parameters': {
 231                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 232                         'x': {"value": {
 233                             'basename': 'blorp.txt',
 234                             'class': 'File',
 235                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 236                             "size": 16
 237                         }},
 238                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 239                               'listing': [
 240                                   {
 241                                       'basename': 'renamed.txt',
 242                                       'class': 'File', 'location':
 243                                       'keep:99999999999999999999999999999998+99/file1.txt',
 244                                       'size': 0
 245                                   }
 246                               ]}},
 247                         'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 248                         'arv:debug': True,
 249                         'arv:enable_reuse': True,
 250                         'arv:on_error': 'continue'
 251                     },
 252                     'repository': 'arvados',
 253                     'script_version': 'master',
 254                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 255                     'script': 'cwl-runner',
 256                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 257                 }
 258             }
 259         }
 260         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 261         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 262         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 263         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 264         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 265             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 266             "state": "Queued"
 267         }
 268         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 269         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 270
 271         with open("tests/wf/submit_wf_packed.cwl") as f:
 272             expect_packed_workflow = yaml.round_trip_load(f)
 273
 274         stubs.expect_container_spec = {
 275             'priority': 500,
 276             'mounts': {
 277                 '/var/spool/cwl': {
 278                     'writable': True,
 279                     'kind': 'collection'
 280                 },
 281                 '/var/lib/cwl/workflow.json': {
 282                     'content': expect_packed_workflow,
 283                     'kind': 'json'
 284                 },
 285                 'stdout': {
 286                     'path': '/var/spool/cwl/cwl.output.json',
 287                     'kind': 'file'
 288                 },
 289                 '/var/lib/cwl/cwl.input.json': {
 290                     'kind': 'json',
 291                     'content': {
 292                         'y': {
 293                             'basename': '99999999999999999999999999999998+99',
 294                             'location': 'keep:99999999999999999999999999999998+99',
 295                             'class': 'Directory'},
 296                         'x': {
 297                             'basename': u'blorp.txt',
 298                             'class': 'File',
 299                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 300                             "size": 16
 301                         },
 302                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 303                             {'basename': 'renamed.txt',
 304                              'class': 'File',
 305                              'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 306                              'size': 0
 307                             }
 308                         ]}
 309                     },
 310                     'kind': 'json'
 311                 }
 312             },
 313             'secret_mounts': {},
 314             'state': 'Committed',
 315             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 316                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 317                         '--eval-timeout=20', '--thread-count=0',
 318                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 319                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 320             'name': 'submit_wf.cwl',
 321             'container_image': '999999999999999999999999999999d3+99',
 322             'output_path': '/var/spool/cwl',
 323             'cwd': '/var/spool/cwl',
 324             'runtime_constraints': {
 325                 'API': True,
 326                 'vcpus': 1,
 327                 'ram': (1024+256)*1024*1024
 328             },
 329             'use_existing': False,
 330             'properties': {},
 331             'secret_mounts': {}
 332         }
 333
 334         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 335         stubs.api.workflows().create().execute.return_value = {
 336             "uuid": stubs.expect_workflow_uuid,
 337         }
 338         def update_mock(**kwargs):
 339             stubs.updated_uuid = kwargs.get('uuid')
 340             return mock.DEFAULT
 341         stubs.api.workflows().update.side_effect = update_mock
 342         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 343             "uuid": stubs.updated_uuid,
 344         }
 345
 346         return func(self, stubs, *args, **kwargs)
 347     return wrapped
 348
 349
 350 class TestSubmit(unittest.TestCase):
 351
 352     def setUp(self):
 353         cwltool.process._names = set()
 354         arvados_cwl.arvdocker.arv_docker_clear_cache()
 355
 356
 357     @mock.patch("time.sleep")
 358     @stubs
 359     def test_submit_invalid_runner_ram(self, stubs, tm):
 360         exited = arvados_cwl.main(
 361             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 362              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 363             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 364         self.assertEqual(exited, 1)
 365
 366
 367     @stubs
 368     def test_submit_container(self, stubs):
 369         exited = arvados_cwl.main(
 370             ["--submit", "--no-wait", "--api=containers", "--debug",
 371                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 372             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 373
 374         stubs.api.collections().create.assert_has_calls([
 375             mock.call(body=JsonDiffMatcher({
 376                 'manifest_text':
 377                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 378                 'replication_desired': None,
 379                 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)',
 380             }), ensure_unique_name=False),
 381             mock.call(body=JsonDiffMatcher({
 382                 'manifest_text':
 383                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 384                 'replication_desired': None,
 385                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 386             }), ensure_unique_name=False),
 387             ])
 388
 389         expect_container = copy.deepcopy(stubs.expect_container_spec)
 390         stubs.api.container_requests().create.assert_called_with(
 391             body=JsonDiffMatcher(expect_container))
 392         self.assertEqual(stubs.capture_stdout.getvalue(),
 393                          stubs.expect_container_request_uuid + '\n')
 394         self.assertEqual(exited, 0)
 395
 396
 397     @stubs
 398     def test_submit_container_tool(self, stubs):
 399         # test for issue #16139
 400         exited = arvados_cwl.main(
 401             ["--submit", "--no-wait", "--api=containers", "--debug",
 402                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 403             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 404
 405         self.assertEqual(stubs.capture_stdout.getvalue(),
 406                          stubs.expect_container_request_uuid + '\n')
 407         self.assertEqual(exited, 0)
 408
 409     @stubs
 410     def test_submit_container_no_reuse(self, stubs):
 411         exited = arvados_cwl.main(
 412             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 413                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 414             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 415
 416         expect_container = copy.deepcopy(stubs.expect_container_spec)
 417         expect_container["command"] = [
 418             'arvados-cwl-runner', '--local', '--api=containers',
 419             '--no-log-timestamps', '--disable-validate', '--disable-color',
 420             '--eval-timeout=20', '--thread-count=0',
 421             '--disable-reuse', "--collection-cache-size=256",
 422             '--debug', '--on-error=continue',
 423             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 424         expect_container["use_existing"] = False
 425
 426         stubs.api.container_requests().create.assert_called_with(
 427             body=JsonDiffMatcher(expect_container))
 428         self.assertEqual(stubs.capture_stdout.getvalue(),
 429                          stubs.expect_container_request_uuid + '\n')
 430         self.assertEqual(exited, 0)
 431
 432     @stubs
 433     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 434         exited = arvados_cwl.main(
 435             ["--submit", "--no-wait", "--api=containers", "--debug",
 436              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 437             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 438         self.assertEqual(exited, 0)
 439
 440         expect_container = copy.deepcopy(stubs.expect_container_spec)
 441         expect_container["command"] = [
 442             'arvados-cwl-runner', '--local', '--api=containers',
 443             '--no-log-timestamps', '--disable-validate', '--disable-color',
 444             '--eval-timeout=20', '--thread-count=0',
 445             '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 446             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 447         expect_container["use_existing"] = False
 448         expect_container["name"] = "submit_wf_no_reuse.cwl"
 449         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 450             {
 451                 "class": "http://arvados.org/cwl#ReuseRequirement",
 452                 "enableReuse": False,
 453             },
 454         ]
 455         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 456             "arv": "http://arvados.org/cwl#",
 457             "cwltool": "http://commonwl.org/cwltool#"
 458         }
 459
 460         stubs.api.container_requests().create.assert_called_with(
 461             body=JsonDiffMatcher(expect_container))
 462         self.assertEqual(stubs.capture_stdout.getvalue(),
 463                          stubs.expect_container_request_uuid + '\n')
 464
 465
 466     @stubs
 467     def test_submit_container_on_error(self, stubs):
 468         exited = arvados_cwl.main(
 469             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 470                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 471             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 472
 473         expect_container = copy.deepcopy(stubs.expect_container_spec)
 474         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 475                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 476                                        '--eval-timeout=20', '--thread-count=0',
 477                                        '--enable-reuse', "--collection-cache-size=256",
 478                                        '--debug', '--on-error=stop',
 479                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 480
 481         stubs.api.container_requests().create.assert_called_with(
 482             body=JsonDiffMatcher(expect_container))
 483         self.assertEqual(stubs.capture_stdout.getvalue(),
 484                          stubs.expect_container_request_uuid + '\n')
 485         self.assertEqual(exited, 0)
 486
 487     @stubs
 488     def test_submit_container_output_name(self, stubs):
 489         output_name = "test_output_name"
 490
 491         exited = arvados_cwl.main(
 492             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 493                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 494             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 495
 496         expect_container = copy.deepcopy(stubs.expect_container_spec)
 497         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 498                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 499                                        '--eval-timeout=20', '--thread-count=0',
 500                                        '--enable-reuse', "--collection-cache-size=256",
 501                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 502                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 503         expect_container["output_name"] = output_name
 504
 505         stubs.api.container_requests().create.assert_called_with(
 506             body=JsonDiffMatcher(expect_container))
 507         self.assertEqual(stubs.capture_stdout.getvalue(),
 508                          stubs.expect_container_request_uuid + '\n')
 509         self.assertEqual(exited, 0)
 510
 511     @stubs
 512     def test_submit_storage_classes(self, stubs):
 513         exited = arvados_cwl.main(
 514             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 515                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 516             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 517
 518         expect_container = copy.deepcopy(stubs.expect_container_spec)
 519         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 520                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 521                                        '--eval-timeout=20', '--thread-count=0',
 522                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 523                                        "--storage-classes=foo", '--on-error=continue',
 524                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 525
 526         stubs.api.container_requests().create.assert_called_with(
 527             body=JsonDiffMatcher(expect_container))
 528         self.assertEqual(stubs.capture_stdout.getvalue(),
 529                          stubs.expect_container_request_uuid + '\n')
 530         self.assertEqual(exited, 0)
 531
 532     @stubs
 533     def test_submit_multiple_storage_classes(self, stubs):
 534         exited = arvados_cwl.main(
 535             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 536                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 537             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 538
 539         expect_container = copy.deepcopy(stubs.expect_container_spec)
 540         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 541                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 542                                        '--eval-timeout=20', '--thread-count=0',
 543                                        '--enable-reuse', "--collection-cache-size=256", "--debug",
 544                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 545                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 546
 547         stubs.api.container_requests().create.assert_called_with(
 548             body=JsonDiffMatcher(expect_container))
 549         self.assertEqual(stubs.capture_stdout.getvalue(),
 550                          stubs.expect_container_request_uuid + '\n')
 551         self.assertEqual(exited, 0)
 552
 553     @mock.patch("cwltool.task_queue.TaskQueue")
 554     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 555     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 556     @stubs
 557     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 558         final_output_c = arvados.collection.Collection()
 559         make_output.return_value = ({},final_output_c)
 560
 561         def set_final_output(job_order, output_callback, runtimeContext):
 562             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 563             return []
 564         job.side_effect = set_final_output
 565
 566         exited = arvados_cwl.main(
 567             ["--debug", "--local", "--storage-classes=foo",
 568                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 569             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 570
 571         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 572         self.assertEqual(exited, 0)
 573
 574     @mock.patch("cwltool.task_queue.TaskQueue")
 575     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 576     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 577     @stubs
 578     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 579         final_output_c = arvados.collection.Collection()
 580         make_output.return_value = ({},final_output_c)
 581         stubs.api.config().get.return_value = {"default": {"Default": True}}
 582
 583         def set_final_output(job_order, output_callback, runtimeContext):
 584             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 585             return []
 586         job.side_effect = set_final_output
 587
 588         exited = arvados_cwl.main(
 589             ["--debug", "--local",
 590                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 591             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 592
 593         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 594         self.assertEqual(exited, 0)
 595
 596     @mock.patch("cwltool.task_queue.TaskQueue")
 597     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 598     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 599     @stubs
 600     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 601         final_output_c = arvados.collection.Collection()
 602         make_output.return_value = ({},final_output_c)
 603
 604         def set_final_output(job_order, output_callback, runtimeContext):
 605             output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
 606             return []
 607         job.side_effect = set_final_output
 608
 609         exited = arvados_cwl.main(
 610             ["--debug", "--local",
 611                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 612             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 613
 614         make_output.assert_called_with(u'Output of submit_storage_class_wf.cwl', ['foo', 'bar'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
 615         self.assertEqual(exited, 0)
 616
 617     @stubs
 618     def test_submit_container_output_ttl(self, stubs):
 619         exited = arvados_cwl.main(
 620             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 621                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 622             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 623
 624         expect_container = copy.deepcopy(stubs.expect_container_spec)
 625         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 626                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 627                                        '--eval-timeout=20', '--thread-count=0',
 628                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 629                                        '--on-error=continue',
 630                                        "--intermediate-output-ttl=3600",
 631                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 632
 633         stubs.api.container_requests().create.assert_called_with(
 634             body=JsonDiffMatcher(expect_container))
 635         self.assertEqual(stubs.capture_stdout.getvalue(),
 636                          stubs.expect_container_request_uuid + '\n')
 637         self.assertEqual(exited, 0)
 638
 639     @stubs
 640     def test_submit_container_trash_intermediate(self, stubs):
 641         exited = arvados_cwl.main(
 642             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 643                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 645
 646
 647         expect_container = copy.deepcopy(stubs.expect_container_spec)
 648         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 649                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 650                                        '--eval-timeout=20', '--thread-count=0',
 651                                        '--enable-reuse', "--collection-cache-size=256",
 652                                        '--debug', '--on-error=continue',
 653                                        "--trash-intermediate",
 654                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 655
 656         stubs.api.container_requests().create.assert_called_with(
 657             body=JsonDiffMatcher(expect_container))
 658         self.assertEqual(stubs.capture_stdout.getvalue(),
 659                          stubs.expect_container_request_uuid + '\n')
 660         self.assertEqual(exited, 0)
 661
 662     @stubs
 663     def test_submit_container_output_tags(self, stubs):
 664         output_tags = "tag0,tag1,tag2"
 665
 666         exited = arvados_cwl.main(
 667             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 668                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 669             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 670
 671         expect_container = copy.deepcopy(stubs.expect_container_spec)
 672         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 673                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 674                                        '--eval-timeout=20', '--thread-count=0',
 675                                        '--enable-reuse', "--collection-cache-size=256",
 676                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 677                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 678
 679         stubs.api.container_requests().create.assert_called_with(
 680             body=JsonDiffMatcher(expect_container))
 681         self.assertEqual(stubs.capture_stdout.getvalue(),
 682                          stubs.expect_container_request_uuid + '\n')
 683         self.assertEqual(exited, 0)
 684
 685     @stubs
 686     def test_submit_container_runner_ram(self, stubs):
 687         exited = arvados_cwl.main(
 688             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 689                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 690             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 691
 692         expect_container = copy.deepcopy(stubs.expect_container_spec)
 693         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 694
 695         stubs.api.container_requests().create.assert_called_with(
 696             body=JsonDiffMatcher(expect_container))
 697         self.assertEqual(stubs.capture_stdout.getvalue(),
 698                          stubs.expect_container_request_uuid + '\n')
 699         self.assertEqual(exited, 0)
 700
 701     @mock.patch("arvados.collection.CollectionReader")
 702     @mock.patch("time.sleep")
 703     @stubs
 704     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 705         collectionReader().exists.return_value = True
 706         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 707         exited = arvados_cwl.main(
 708             ["--submit", "--no-wait", "--api=containers", "--debug",
 709              "tests/wf/submit_keepref_wf.cwl"],
 710             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 711         self.assertEqual(exited, 0)
 712
 713     @mock.patch("arvados.collection.CollectionReader")
 714     @mock.patch("time.sleep")
 715     @stubs
 716     def test_submit_keepref(self, stubs, tm, reader):
 717         with open("tests/wf/expect_arvworkflow.cwl") as f:
 718             reader().open().__enter__().read.return_value = f.read()
 719
 720         exited = arvados_cwl.main(
 721             ["--submit", "--no-wait", "--api=containers", "--debug",
 722              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 723             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 724
 725         expect_container = {
 726             'priority': 500,
 727             'mounts': {
 728                 '/var/spool/cwl': {
 729                     'writable': True,
 730                     'kind': 'collection'
 731                 },
 732                 'stdout': {
 733                     'path': '/var/spool/cwl/cwl.output.json',
 734                     'kind': 'file'
 735                 },
 736                 '/var/lib/cwl/workflow': {
 737                     'portable_data_hash': '99999999999999999999999999999994+99',
 738                     'kind': 'collection'
 739                 },
 740                 '/var/lib/cwl/cwl.input.json': {
 741                     'content': {
 742                         'x': 'XxX'
 743                     },
 744                     'kind': 'json'
 745                 }
 746             }, 'state': 'Committed',
 747             'output_path': '/var/spool/cwl',
 748             'name': 'expect_arvworkflow.cwl#main',
 749             'container_image': '999999999999999999999999999999d3+99',
 750             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 751                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 752                         '--eval-timeout=20', '--thread-count=0',
 753                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 754                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 755             'cwd': '/var/spool/cwl',
 756             'runtime_constraints': {
 757                 'API': True,
 758                 'vcpus': 1,
 759                 'ram': 1342177280
 760             },
 761             'use_existing': False,
 762             'properties': {},
 763             'secret_mounts': {}
 764         }
 765
 766         stubs.api.container_requests().create.assert_called_with(
 767             body=JsonDiffMatcher(expect_container))
 768         self.assertEqual(stubs.capture_stdout.getvalue(),
 769                          stubs.expect_container_request_uuid + '\n')
 770         self.assertEqual(exited, 0)
 771
 772     @mock.patch("time.sleep")
 773     @stubs
 774     def test_submit_arvworkflow(self, stubs, tm):
 775         with open("tests/wf/expect_arvworkflow.cwl") as f:
 776             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 777
 778         exited = arvados_cwl.main(
 779             ["--submit", "--no-wait", "--api=containers", "--debug",
 780              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 781             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 782
 783         expect_container = {
 784             'priority': 500,
 785             'mounts': {
 786                 '/var/spool/cwl': {
 787                     'writable': True,
 788                     'kind': 'collection'
 789                 },
 790                 'stdout': {
 791                     'path': '/var/spool/cwl/cwl.output.json',
 792                     'kind': 'file'
 793                 },
 794                 '/var/lib/cwl/workflow.json': {
 795                     'kind': 'json',
 796                     'content': {
 797                         'cwlVersion': 'v1.0',
 798                         '$graph': [
 799                             {
 800                                 'id': '#main',
 801                                 'inputs': [
 802                                     {'type': 'string', 'id': '#main/x'}
 803                                 ],
 804                                 'steps': [
 805                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 806                                      'run': '#submit_tool.cwl',
 807                                      'id': '#main/step1',
 808                                      'out': []}
 809                                 ],
 810                                 'class': 'Workflow',
 811                                 'outputs': []
 812                             },
 813                             {
 814                                 'inputs': [
 815                                     {
 816                                         'inputBinding': {'position': 1},
 817                                         'type': 'string',
 818                                         'id': '#submit_tool.cwl/x'}
 819                                 ],
 820                                 'requirements': [
 821                                     {
 822                                         'dockerPull': 'debian:buster-slim',
 823                                         'class': 'DockerRequirement',
 824                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
 825                                     }
 826                                 ],
 827                                 'id': '#submit_tool.cwl',
 828                                 'outputs': [],
 829                                 'baseCommand': 'cat',
 830                                 'class': 'CommandLineTool'
 831                             }
 832                         ]
 833                     }
 834                 },
 835                 '/var/lib/cwl/cwl.input.json': {
 836                     'content': {
 837                         'x': 'XxX'
 838                     },
 839                     'kind': 'json'
 840                 }
 841             }, 'state': 'Committed',
 842             'output_path': '/var/spool/cwl',
 843             'name': 'a test workflow',
 844             'container_image': "999999999999999999999999999999d3+99",
 845             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 846                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 847                         '--eval-timeout=20', '--thread-count=0',
 848                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
 849                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 850             'cwd': '/var/spool/cwl',
 851             'runtime_constraints': {
 852                 'API': True,
 853                 'vcpus': 1,
 854                 'ram': 1342177280
 855             },
 856             'use_existing': False,
 857             'properties': {
 858                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 859             },
 860             'secret_mounts': {}
 861         }
 862
 863         stubs.api.container_requests().create.assert_called_with(
 864             body=JsonDiffMatcher(expect_container))
 865         self.assertEqual(stubs.capture_stdout.getvalue(),
 866                          stubs.expect_container_request_uuid + '\n')
 867         self.assertEqual(exited, 0)
 868
 869     @stubs
 870     def test_submit_container_name(self, stubs):
 871         exited = arvados_cwl.main(
 872             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 873                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 874             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 875
 876         expect_container = copy.deepcopy(stubs.expect_container_spec)
 877         expect_container["name"] = "hello container 123"
 878
 879         stubs.api.container_requests().create.assert_called_with(
 880             body=JsonDiffMatcher(expect_container))
 881         self.assertEqual(stubs.capture_stdout.getvalue(),
 882                          stubs.expect_container_request_uuid + '\n')
 883         self.assertEqual(exited, 0)
 884
 885     @stubs
 886     def test_submit_missing_input(self, stubs):
 887         exited = arvados_cwl.main(
 888             ["--submit", "--no-wait", "--api=containers", "--debug",
 889              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 890             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 891         self.assertEqual(exited, 0)
 892
 893         exited = arvados_cwl.main(
 894             ["--submit", "--no-wait", "--api=containers", "--debug",
 895              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 896             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 897         self.assertEqual(exited, 1)
 898
 899     @stubs
 900     def test_submit_container_project(self, stubs):
 901         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 902         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 903         exited = arvados_cwl.main(
 904             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 905                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 906             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 907
 908         expect_container = copy.deepcopy(stubs.expect_container_spec)
 909         expect_container["owner_uuid"] = project_uuid
 910         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 911                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 912                                        "--eval-timeout=20", "--thread-count=0",
 913                                        '--enable-reuse', "--collection-cache-size=256", '--debug',
 914                                        '--on-error=continue',
 915                                        '--project-uuid='+project_uuid,
 916                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 917
 918         stubs.api.container_requests().create.assert_called_with(
 919             body=JsonDiffMatcher(expect_container))
 920         self.assertEqual(stubs.capture_stdout.getvalue(),
 921                          stubs.expect_container_request_uuid + '\n')
 922         self.assertEqual(exited, 0)
 923
 924     @stubs
 925     def test_submit_container_eval_timeout(self, stubs):
 926         exited = arvados_cwl.main(
 927             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 928                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 929             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 930
 931         expect_container = copy.deepcopy(stubs.expect_container_spec)
 932         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 933                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 934                                        '--eval-timeout=60.0', '--thread-count=0',
 935                                        '--enable-reuse', "--collection-cache-size=256",
 936                                        '--debug', '--on-error=continue',
 937                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 938
 939         stubs.api.container_requests().create.assert_called_with(
 940             body=JsonDiffMatcher(expect_container))
 941         self.assertEqual(stubs.capture_stdout.getvalue(),
 942                          stubs.expect_container_request_uuid + '\n')
 943         self.assertEqual(exited, 0)
 944
 945     @stubs
 946     def test_submit_container_collection_cache(self, stubs):
 947         exited = arvados_cwl.main(
 948             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 949                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 950             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 951
 952         expect_container = copy.deepcopy(stubs.expect_container_spec)
 953         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 954                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 955                                        '--eval-timeout=20', '--thread-count=0',
 956                                        '--enable-reuse', "--collection-cache-size=500",
 957                                        '--debug', '--on-error=continue',
 958                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 959         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 960
 961         stubs.api.container_requests().create.assert_called_with(
 962             body=JsonDiffMatcher(expect_container))
 963         self.assertEqual(stubs.capture_stdout.getvalue(),
 964                          stubs.expect_container_request_uuid + '\n')
 965         self.assertEqual(exited, 0)
 966
 967     @stubs
 968     def test_submit_container_thread_count(self, stubs):
 969         exited = arvados_cwl.main(
 970             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 971                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 972             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 973
 974         expect_container = copy.deepcopy(stubs.expect_container_spec)
 975         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 976                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 977                                        '--eval-timeout=20', '--thread-count=20',
 978                                        '--enable-reuse', "--collection-cache-size=256",
 979                                        '--debug', '--on-error=continue',
 980                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 981
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     @stubs
 989     def test_submit_container_runner_image(self, stubs):
 990         exited = arvados_cwl.main(
 991             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 992                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 993             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 994
 995         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 996
 997         expect_container = copy.deepcopy(stubs.expect_container_spec)
 998         stubs.api.container_requests().create.assert_called_with(
 999             body=JsonDiffMatcher(expect_container))
1000         self.assertEqual(stubs.capture_stdout.getvalue(),
1001                          stubs.expect_container_request_uuid + '\n')
1002         self.assertEqual(exited, 0)
1003
1004     @stubs
1005     def test_submit_priority(self, stubs):
1006         exited = arvados_cwl.main(
1007             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1008                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1009             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1010
1011         stubs.expect_container_spec["priority"] = 669
1012
1013         expect_container = copy.deepcopy(stubs.expect_container_spec)
1014         stubs.api.container_requests().create.assert_called_with(
1015             body=JsonDiffMatcher(expect_container))
1016         self.assertEqual(stubs.capture_stdout.getvalue(),
1017                          stubs.expect_container_request_uuid + '\n')
1018         self.assertEqual(exited, 0)
1019
1020     @stubs
1021     def test_submit_wf_runner_resources(self, stubs):
1022         exited = arvados_cwl.main(
1023             ["--submit", "--no-wait", "--api=containers", "--debug",
1024                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1025             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1026
1027         expect_container = copy.deepcopy(stubs.expect_container_spec)
1028         expect_container["runtime_constraints"] = {
1029             "API": True,
1030             "vcpus": 2,
1031             "ram": (2000+512) * 2**20
1032         }
1033         expect_container["name"] = "submit_wf_runner_resources.cwl"
1034         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1035             {
1036                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1037                 "coresMin": 2,
1038                 "ramMin": 2000,
1039                 "keep_cache": 512
1040             }
1041         ]
1042         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1043             "arv": "http://arvados.org/cwl#",
1044         }
1045         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1046                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1047                         '--eval-timeout=20', '--thread-count=0',
1048                         '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
1049                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1050
1051         stubs.api.container_requests().create.assert_called_with(
1052             body=JsonDiffMatcher(expect_container))
1053         self.assertEqual(stubs.capture_stdout.getvalue(),
1054                          stubs.expect_container_request_uuid + '\n')
1055         self.assertEqual(exited, 0)
1056
1057     def tearDown(self):
1058         arvados_cwl.arvdocker.arv_docker_clear_cache()
1059
1060     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1061     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1062     @mock.patch("arvados.api")
1063     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1064         arvados_cwl.arvdocker.arv_docker_clear_cache()
1065
1066         arvrunner = mock.MagicMock()
1067         arvrunner.project_uuid = ""
1068         api.return_value = mock.MagicMock()
1069         arvrunner.api = api.return_value
1070         arvrunner.runtimeContext.match_local_docker = False
1071         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1072                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1073                                                                         "link_class": "docker_image_repo+tag",
1074                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1075                                                                         "owner_uuid": "",
1076                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1077                                                             {"items": [{"created_at": "",
1078                                                                         "head_uuid": "",
1079                                                                         "link_class": "docker_image_hash",
1080                                                                         "name": "123456",
1081                                                                         "owner_uuid": "",
1082                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1083         )
1084         find_one_image_hash.return_value = "123456"
1085
1086         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1087                                                                               "owner_uuid": "",
1088                                                                               "manifest_text": "",
1089                                                                               "properties": ""
1090                                                                           }], "items_available": 1, "offset": 0},)
1091         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1092         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1093                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1094         self.assertEqual("9999999999999999999999999999999b+99",
1095                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
1096
1097
1098     @stubs
1099     def test_submit_secrets(self, stubs):
1100         exited = arvados_cwl.main(
1101             ["--submit", "--no-wait", "--api=containers", "--debug",
1102                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1103             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1104
1105         expect_container = {
1106             "command": [
1107                 "arvados-cwl-runner",
1108                 "--local",
1109                 "--api=containers",
1110                 "--no-log-timestamps",
1111                 "--disable-validate",
1112                 "--disable-color",
1113                 "--eval-timeout=20",
1114                 '--thread-count=0',
1115                 "--enable-reuse",
1116                 "--collection-cache-size=256",
1117                 '--debug',
1118                 "--on-error=continue",
1119                 "/var/lib/cwl/workflow.json#main",
1120                 "/var/lib/cwl/cwl.input.json"
1121             ],
1122             "container_image": "999999999999999999999999999999d3+99",
1123             "cwd": "/var/spool/cwl",
1124             "mounts": {
1125                 "/var/lib/cwl/cwl.input.json": {
1126                     "content": {
1127                         "pw": {
1128                             "$include": "/secrets/s0"
1129                         }
1130                     },
1131                     "kind": "json"
1132                 },
1133                 "/var/lib/cwl/workflow.json": {
1134                     "content": {
1135                         "$graph": [
1136                             {
1137                                 "arguments": [
1138                                     "md5sum",
1139                                     "example.conf"
1140                                 ],
1141                                 "class": "CommandLineTool",
1142                                 "hints": [
1143                                     {
1144                                         "class": "http://commonwl.org/cwltool#Secrets",
1145                                         "secrets": [
1146                                             "#secret_job.cwl/pw"
1147                                         ]
1148                                     }
1149                                 ],
1150                                 "id": "#secret_job.cwl",
1151                                 "inputs": [
1152                                     {
1153                                         "id": "#secret_job.cwl/pw",
1154                                         "type": "string"
1155                                     }
1156                                 ],
1157                                 "outputs": [
1158                                     {
1159                                         "id": "#secret_job.cwl/out",
1160                                         "type": "File",
1161                                         "outputBinding": {
1162                                               "glob": "hashed_example.txt"
1163                                         }
1164                                     }
1165                                 ],
1166                                 "stdout": "hashed_example.txt",
1167                                 "requirements": [
1168                                     {
1169                                         "class": "InitialWorkDirRequirement",
1170                                         "listing": [
1171                                             {
1172                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1173                                                 "entryname": "example.conf"
1174                                             }
1175                                         ]
1176                                     }
1177                                 ]
1178                             },
1179                             {
1180                                 "class": "Workflow",
1181                                 "hints": [
1182                                     {
1183                                         "class": "DockerRequirement",
1184                                         "dockerPull": "debian:buster-slim",
1185                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1186                                     },
1187                                     {
1188                                         "class": "http://commonwl.org/cwltool#Secrets",
1189                                         "secrets": [
1190                                             "#main/pw"
1191                                         ]
1192                                     }
1193                                 ],
1194                                 "id": "#main",
1195                                 "inputs": [
1196                                     {
1197                                         "id": "#main/pw",
1198                                         "type": "string"
1199                                     }
1200                                 ],
1201                                 "outputs": [
1202                                     {
1203                                         "id": "#main/out",
1204                                         "outputSource": "#main/step1/out",
1205                                         "type": "File"
1206                                     }
1207                                 ],
1208                                 "steps": [
1209                                     {
1210                                         "id": "#main/step1",
1211                                         "in": [
1212                                             {
1213                                                 "id": "#main/step1/pw",
1214                                                 "source": "#main/pw"
1215                                             }
1216                                         ],
1217                                         "out": [
1218                                             "#main/step1/out"
1219                                         ],
1220                                         "run": "#secret_job.cwl"
1221                                     }
1222                                 ]
1223                             }
1224                         ],
1225                         "$namespaces": {
1226                             "cwltool": "http://commonwl.org/cwltool#"
1227                         },
1228                         "cwlVersion": "v1.0"
1229                     },
1230                     "kind": "json"
1231                 },
1232                 "/var/spool/cwl": {
1233                     "kind": "collection",
1234                     "writable": True
1235                 },
1236                 "stdout": {
1237                     "kind": "file",
1238                     "path": "/var/spool/cwl/cwl.output.json"
1239                 }
1240             },
1241             "name": "secret_wf.cwl",
1242             "output_path": "/var/spool/cwl",
1243             "priority": 500,
1244             "properties": {},
1245             "runtime_constraints": {
1246                 "API": True,
1247                 "ram": 1342177280,
1248                 "vcpus": 1
1249             },
1250             "secret_mounts": {
1251                 "/secrets/s0": {
1252                     "content": "blorp",
1253                     "kind": "text"
1254                 }
1255             },
1256             "state": "Committed",
1257             "use_existing": False
1258         }
1259
1260         stubs.api.container_requests().create.assert_called_with(
1261             body=JsonDiffMatcher(expect_container))
1262         self.assertEqual(stubs.capture_stdout.getvalue(),
1263                          stubs.expect_container_request_uuid + '\n')
1264         self.assertEqual(exited, 0)
1265
1266     @stubs
1267     def test_submit_request_uuid(self, stubs):
1268         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1269         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1270
1271         stubs.api.container_requests().update().execute.return_value = {
1272             "uuid": stubs.expect_container_request_uuid,
1273             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1274             "state": "Queued"
1275         }
1276
1277         exited = arvados_cwl.main(
1278             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1279                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1280             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1281
1282         stubs.api.container_requests().update.assert_called_with(
1283             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1284         self.assertEqual(stubs.capture_stdout.getvalue(),
1285                          stubs.expect_container_request_uuid + '\n')
1286         self.assertEqual(exited, 0)
1287
1288     @stubs
1289     def test_submit_container_cluster_id(self, stubs):
1290         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1291
1292         exited = arvados_cwl.main(
1293             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1294                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1295             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1296
1297         expect_container = copy.deepcopy(stubs.expect_container_spec)
1298
1299         stubs.api.container_requests().create.assert_called_with(
1300             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1301         self.assertEqual(stubs.capture_stdout.getvalue(),
1302                          stubs.expect_container_request_uuid + '\n')
1303         self.assertEqual(exited, 0)
1304
1305     @stubs
1306     def test_submit_validate_cluster_id(self, stubs):
1307         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1308         exited = arvados_cwl.main(
1309             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1310              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1311             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1312         self.assertEqual(exited, 1)
1313
1314     @stubs
1315     def test_submit_validate_project_uuid(self, stubs):
1316         # Fails with bad cluster prefix
1317         exited = arvados_cwl.main(
1318             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1320             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1321         self.assertEqual(exited, 1)
1322
1323         # Project lookup fails
1324         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1325         exited = arvados_cwl.main(
1326             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1327              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1328             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1329         self.assertEqual(exited, 1)
1330
1331         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1332         exited = arvados_cwl.main(
1333             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1334              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1335             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1336         self.assertEqual(exited, 0)
1337
1338
1339     @mock.patch("arvados.collection.CollectionReader")
1340     @stubs
1341     def test_submit_uuid_inputs(self, stubs, collectionReader):
1342         collectionReader().exists.return_value = True
1343         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1344         def list_side_effect(**kwargs):
1345             m = mock.MagicMock()
1346             if "count" in kwargs:
1347                 m.execute.return_value = {"items": [
1348                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1349                 ]}
1350             else:
1351                 m.execute.return_value = {"items": []}
1352             return m
1353         stubs.api.collections().list.side_effect = list_side_effect
1354
1355         exited = arvados_cwl.main(
1356             ["--submit", "--no-wait", "--api=containers", "--debug",
1357                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1358             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1359
1360         expect_container = copy.deepcopy(stubs.expect_container_spec)
1361         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1362         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1363         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1364
1365         stubs.api.collections().list.assert_has_calls([
1366             mock.call(count='none',
1367                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1368                       select=['uuid', 'portable_data_hash'])])
1369         stubs.api.container_requests().create.assert_called_with(
1370             body=JsonDiffMatcher(expect_container))
1371         self.assertEqual(stubs.capture_stdout.getvalue(),
1372                          stubs.expect_container_request_uuid + '\n')
1373         self.assertEqual(exited, 0)
1374
1375     @stubs
1376     def test_submit_mismatched_uuid_inputs(self, stubs):
1377         def list_side_effect(**kwargs):
1378             m = mock.MagicMock()
1379             if "count" in kwargs:
1380                 m.execute.return_value = {"items": [
1381                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1382                 ]}
1383             else:
1384                 m.execute.return_value = {"items": []}
1385             return m
1386         stubs.api.collections().list.side_effect = list_side_effect
1387
1388         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1389             capture_stderr = StringIO()
1390             cwltool_logger = logging.getLogger('cwltool')
1391             stderr_logger = logging.StreamHandler(capture_stderr)
1392             cwltool_logger.addHandler(stderr_logger)
1393
1394             try:
1395                 exited = arvados_cwl.main(
1396                     ["--submit", "--no-wait", "--api=containers", "--debug",
1397                         "tests/wf/submit_wf.cwl", infile],
1398                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1399
1400                 self.assertEqual(exited, 1)
1401                 self.assertRegex(
1402                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1403                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1404             finally:
1405                 cwltool_logger.removeHandler(stderr_logger)
1406
1407     @mock.patch("arvados.collection.CollectionReader")
1408     @stubs
1409     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1410         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1411         capture_stderr = StringIO()
1412
1413         cwltool_logger = logging.getLogger('cwltool')
1414         stderr_logger = logging.StreamHandler(capture_stderr)
1415         cwltool_logger.addHandler(stderr_logger)
1416
1417         exited = arvados_cwl.main(
1418             ["--submit", "--no-wait", "--api=containers", "--debug",
1419                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1420             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1421
1422         try:
1423             self.assertEqual(exited, 1)
1424             self.assertRegex(
1425                 capture_stderr.getvalue(),
1426                 r"Collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz not found")
1427         finally:
1428             cwltool_logger.removeHandler(stderr_logger)
1429
1430     @stubs
1431     def test_submit_set_process_properties(self, stubs):
1432         exited = arvados_cwl.main(
1433             ["--submit", "--no-wait", "--api=containers", "--debug",
1434                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1435             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1436
1437         expect_container = copy.deepcopy(stubs.expect_container_spec)
1438         expect_container["name"] = "submit_wf_process_properties.cwl"
1439         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1440             {
1441                 "class": "http://arvados.org/cwl#ProcessProperties",
1442                 "processProperties": [
1443                     {"propertyName": "baz",
1444                      "propertyValue": "$(inputs.x.basename)"},
1445                     {"propertyName": "foo",
1446                      "propertyValue": "bar"},
1447                     {"propertyName": "quux",
1448                      "propertyValue": {
1449                          "q1": 1,
1450                          "q2": 2
1451                      }
1452                     }
1453                 ],
1454             }
1455         ]
1456         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1457             "arv": "http://arvados.org/cwl#"
1458         }
1459
1460         expect_container["properties"] = {
1461             "baz": "blorp.txt",
1462             "foo": "bar",
1463             "quux": {
1464                 "q1": 1,
1465                 "q2": 2
1466             }
1467         }
1468
1469         stubs.api.container_requests().create.assert_called_with(
1470             body=JsonDiffMatcher(expect_container))
1471         self.assertEqual(stubs.capture_stdout.getvalue(),
1472                          stubs.expect_container_request_uuid + '\n')
1473         self.assertEqual(exited, 0)
1474
1475
1476     @stubs
1477     def test_submit_enable_preemptible(self, stubs):
1478         exited = arvados_cwl.main(
1479             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1480                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1481             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1482
1483         expect_container = copy.deepcopy(stubs.expect_container_spec)
1484         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1485                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1486                         '--eval-timeout=20', '--thread-count=0',
1487                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1488                                        '--enable-preemptible',
1489                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1490
1491         stubs.api.container_requests().create.assert_called_with(
1492             body=JsonDiffMatcher(expect_container))
1493         self.assertEqual(stubs.capture_stdout.getvalue(),
1494                          stubs.expect_container_request_uuid + '\n')
1495         self.assertEqual(exited, 0)
1496
1497     @stubs
1498     def test_submit_disable_preemptible(self, stubs):
1499         exited = arvados_cwl.main(
1500             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1501                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1502             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1503
1504         expect_container = copy.deepcopy(stubs.expect_container_spec)
1505         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1506                         '--no-log-timestamps', '--disable-validate', '--disable-color',
1507                         '--eval-timeout=20', '--thread-count=0',
1508                         '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
1509                                        '--disable-preemptible',
1510                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1511
1512         stubs.api.container_requests().create.assert_called_with(
1513             body=JsonDiffMatcher(expect_container))
1514         self.assertEqual(stubs.capture_stdout.getvalue(),
1515                          stubs.expect_container_request_uuid + '\n')
1516         self.assertEqual(exited, 0)
1517
1518
1519 class TestCreateWorkflow(unittest.TestCase):
1520     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1521     expect_workflow = StripYAMLComments(
1522         open("tests/wf/expect_upload_packed.cwl").read().rstrip())
1523
1524     def setUp(self):
1525         cwltool.process._names = set()
1526         arvados_cwl.arvdocker.arv_docker_clear_cache()
1527
1528     @stubs
1529     def test_create(self, stubs):
1530         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1531         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1532
1533         exited = arvados_cwl.main(
1534             ["--create-workflow", "--debug",
1535              "--api=containers",
1536              "--project-uuid", project_uuid,
1537              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1538             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1539
1540         stubs.api.pipeline_templates().create.refute_called()
1541         stubs.api.container_requests().create.refute_called()
1542
1543         body = {
1544             "workflow": {
1545                 "owner_uuid": project_uuid,
1546                 "name": "submit_wf.cwl",
1547                 "description": "",
1548                 "definition": self.expect_workflow,
1549             }
1550         }
1551         stubs.api.workflows().create.assert_called_with(
1552             body=JsonDiffMatcher(body))
1553
1554         self.assertEqual(stubs.capture_stdout.getvalue(),
1555                          stubs.expect_workflow_uuid + '\n')
1556         self.assertEqual(exited, 0)
1557
1558     @stubs
1559     def test_create_name(self, stubs):
1560         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1561         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1562
1563         exited = arvados_cwl.main(
1564             ["--create-workflow", "--debug",
1565              "--api=containers",
1566              "--project-uuid", project_uuid,
1567              "--name", "testing 123",
1568              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1569             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1570
1571         stubs.api.pipeline_templates().create.refute_called()
1572         stubs.api.container_requests().create.refute_called()
1573
1574         body = {
1575             "workflow": {
1576                 "owner_uuid": project_uuid,
1577                 "name": "testing 123",
1578                 "description": "",
1579                 "definition": self.expect_workflow,
1580             }
1581         }
1582         stubs.api.workflows().create.assert_called_with(
1583             body=JsonDiffMatcher(body))
1584
1585         self.assertEqual(stubs.capture_stdout.getvalue(),
1586                          stubs.expect_workflow_uuid + '\n')
1587         self.assertEqual(exited, 0)
1588
1589
1590     @stubs
1591     def test_update(self, stubs):
1592         exited = arvados_cwl.main(
1593             ["--update-workflow", self.existing_workflow_uuid,
1594              "--debug",
1595              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1596             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1597
1598         body = {
1599             "workflow": {
1600                 "name": "submit_wf.cwl",
1601                 "description": "",
1602                 "definition": self.expect_workflow,
1603             }
1604         }
1605         stubs.api.workflows().update.assert_called_with(
1606             uuid=self.existing_workflow_uuid,
1607             body=JsonDiffMatcher(body))
1608         self.assertEqual(stubs.capture_stdout.getvalue(),
1609                          self.existing_workflow_uuid + '\n')
1610         self.assertEqual(exited, 0)
1611
1612
1613     @stubs
1614     def test_update_name(self, stubs):
1615         exited = arvados_cwl.main(
1616             ["--update-workflow", self.existing_workflow_uuid,
1617              "--debug", "--name", "testing 123",
1618              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1619             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1620
1621         body = {
1622             "workflow": {
1623                 "name": "testing 123",
1624                 "description": "",
1625                 "definition": self.expect_workflow,
1626             }
1627         }
1628         stubs.api.workflows().update.assert_called_with(
1629             uuid=self.existing_workflow_uuid,
1630             body=JsonDiffMatcher(body))
1631         self.assertEqual(stubs.capture_stdout.getvalue(),
1632                          self.existing_workflow_uuid + '\n')
1633         self.assertEqual(exited, 0)
1634
1635     @stubs
1636     def test_create_collection_per_tool(self, stubs):
1637         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1638         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1639
1640         exited = arvados_cwl.main(
1641             ["--create-workflow", "--debug",
1642              "--api=containers",
1643              "--project-uuid", project_uuid,
1644              "tests/collection_per_tool/collection_per_tool.cwl"],
1645             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1646
1647         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
1648         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1649
1650         body = {
1651             "workflow": {
1652                 "owner_uuid": project_uuid,
1653                 "name": "collection_per_tool.cwl",
1654                 "description": "",
1655                 "definition": expect_workflow,
1656             }
1657         }
1658         stubs.api.workflows().create.assert_called_with(
1659             body=JsonDiffMatcher(body))
1660
1661         self.assertEqual(stubs.capture_stdout.getvalue(),
1662                          stubs.expect_workflow_uuid + '\n')
1663         self.assertEqual(exited, 0)
1664
1665     @stubs
1666     def test_create_with_imports(self, stubs):
1667         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1668         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1669
1670         exited = arvados_cwl.main(
1671             ["--create-workflow", "--debug",
1672              "--api=containers",
1673              "--project-uuid", project_uuid,
1674              "tests/wf/feddemo/feddemo.cwl"],
1675             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1676
1677         stubs.api.pipeline_templates().create.refute_called()
1678         stubs.api.container_requests().create.refute_called()
1679
1680         self.assertEqual(stubs.capture_stdout.getvalue(),
1681                          stubs.expect_workflow_uuid + '\n')
1682         self.assertEqual(exited, 0)
1683
1684     @stubs
1685     def test_create_with_no_input(self, stubs):
1686         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1687         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1688
1689         exited = arvados_cwl.main(
1690             ["--create-workflow", "--debug",
1691              "--api=containers",
1692              "--project-uuid", project_uuid,
1693              "tests/wf/revsort/revsort.cwl"],
1694             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1695
1696         stubs.api.pipeline_templates().create.refute_called()
1697         stubs.api.container_requests().create.refute_called()
1698
1699         self.assertEqual(stubs.capture_stdout.getvalue(),
1700                          stubs.expect_workflow_uuid + '\n')
1701         self.assertEqual(exited, 0)