sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import io
   7 import itertools
   8 import functools
   9 import hashlib
  10 import json
  11 import logging
  12 import sys
  13 import unittest
  14 import cwltool.process
  15 import re
  16 import os
  17
  18 from io import BytesIO, StringIO
  19 from unittest import mock
  20
  21 import arvados
  22 import arvados.collection
  23 import arvados_cwl
  24 import arvados_cwl.executor
  25 import arvados_cwl.runner
  26 import arvados.keep
  27
  28 from .matcher import JsonDiffMatcher, StripYAMLComments
  29 from .mock_discovery import get_rootDesc
  30
  31 import ruamel.yaml
  32
  33 _rootDesc = None
  34
  35 def stubs(wfdetails=('submit_wf.cwl', None)):
  36     def outer_wrapper(func, *rest):
  37         @functools.wraps(func)
  38         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  39         @mock.patch("uuid.uuid4")
  40         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  41         @mock.patch("arvados.collection.KeepClient")
  42         @mock.patch("arvados.keep.KeepClient")
  43         @mock.patch("arvados.events.subscribe")
  44         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  45                     uuid4, determine_image_id, *args, **kwargs):
  46             class Stubs(object):
  47                 pass
  48
  49             wfname = wfdetails[0]
  50             wfpath = wfdetails[1]
  51
  52             stubs = Stubs()
  53             stubs.events = events
  54             stubs.keepdocker = keepdocker
  55
  56             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  57                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  58                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  59                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  60                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  61                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  62                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  63                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  64                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  65
  66             determine_image_id.return_value = None
  67
  68             def putstub(p, **kwargs):
  69                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  70             keep_client1().put.side_effect = putstub
  71             keep_client1.put.side_effect = putstub
  72             keep_client2().put.side_effect = putstub
  73             keep_client2.put.side_effect = putstub
  74
  75             stubs.keep_client = keep_client2
  76             stubs.docker_images = {
  77                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  78                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  79                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  80                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  81             }
  82             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  83                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  84             stubs.keepdocker.side_effect = kd
  85
  86             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  87             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  88
  89             stubs.capture_stdout = StringIO()
  90
  91             stubs.api = mock.MagicMock()
  92             stubs.api._rootDesc = get_rootDesc()
  93             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  94             stubs.api._rootDesc["revision"] = "20210628"
  95
  96             stubs.api.users().current().execute.return_value = {
  97                 "uuid": stubs.fake_user_uuid,
  98             }
  99             stubs.api.collections().list().execute.return_value = {"items": []}
 100             stubs.api.containers().current().execute.return_value = {
 101                 "uuid": stubs.fake_container_uuid,
 102             }
 103             stubs.api.config()["StorageClasses"].items.return_value = {
 104                 "default": {
 105                     "Default": True
 106                 }
 107             }.items()
 108
 109             class CollectionExecute(object):
 110                 def __init__(self, exe):
 111                     self.exe = exe
 112                 def execute(self, num_retries=None):
 113                     return self.exe
 114
 115             def collection_createstub(created_collections, body, ensure_unique_name=None):
 116                 mt = body["manifest_text"].encode('utf-8')
 117                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 118                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 119                 created_collections[uuid] = {
 120                     "uuid": uuid,
 121                     "portable_data_hash": pdh,
 122                     "manifest_text": mt.decode('utf-8')
 123                 }
 124                 return CollectionExecute(created_collections[uuid])
 125
 126             def collection_getstub(created_collections, uuid):
 127                 for v in created_collections.values():
 128                     if uuid in (v["uuid"], v["portable_data_hash"]):
 129                         return CollectionExecute(v)
 130
 131             created_collections = {
 132                 "99999999999999999999999999999998+99": {
 133                     "uuid": "",
 134                     "portable_data_hash": "99999999999999999999999999999998+99",
 135                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 136                 },
 137                 "99999999999999999999999999999997+99": {
 138                     "uuid": "",
 139                     "portable_data_hash": "99999999999999999999999999999997+99",
 140                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 141                 },
 142                 "99999999999999999999999999999994+99": {
 143                     "uuid": "",
 144                     "portable_data_hash": "99999999999999999999999999999994+99",
 145                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 146                 },
 147                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 148                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 149                     "portable_data_hash": "999999999999999999999999999999d3+99",
 150                     "manifest_text": ""
 151                 },
 152                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 153                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 154                     "portable_data_hash": "999999999999999999999999999999d4+99",
 155                     "manifest_text": ""
 156                 },
 157                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 158                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 159                     "portable_data_hash": "999999999999999999999999999999d5+99",
 160                     "manifest_text": ""
 161                 },
 162                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 163                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 164                     "portable_data_hash": "999999999999999999999999999999d6+99",
 165                     "manifest_text": ""
 166                 }
 167             }
 168             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 169             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 170
 171             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 172             stubs.api.container_requests().create().execute.return_value = {
 173                 "uuid": stubs.expect_container_request_uuid,
 174                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 175                 "state": "Queued"
 176             }
 177
 178             cwd = os.getcwd()
 179             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 180             with open(filepath) as f:
 181                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 182                 expect_packed_workflow = yaml.load(f)
 183
 184             if wfpath is None:
 185                 wfpath = wfname
 186
 187             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 188             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 189             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 190
 191             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 192             expect_packed_workflow.update(stubs.git_info)
 193
 194             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 195
 196             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 197             if wfname == wfpath:
 198                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 199             else:
 200                 container_name = wfname
 201
 202             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 203             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 204
 205             stubs.expect_container_spec = {
 206                 'priority': 500,
 207                 'mounts': {
 208                     '/var/spool/cwl': {
 209                         'writable': True,
 210                         'kind': 'collection'
 211                     },
 212                     '/var/lib/cwl/workflow.json': {
 213                         'content': expect_packed_workflow,
 214                         'kind': 'json'
 215                     },
 216                     'stdout': {
 217                         'path': '/var/spool/cwl/cwl.output.json',
 218                         'kind': 'file'
 219                     },
 220                     '/var/lib/cwl/cwl.input.json': {
 221                         'kind': 'json',
 222                         'content': {
 223                             'y': {
 224                                 'basename': '99999999999999999999999999999998+99',
 225                                 'location': 'keep:99999999999999999999999999999998+99',
 226                                 'class': 'Directory'},
 227                             'x': {
 228                                 'basename': u'blorp.txt',
 229                                 'class': 'File',
 230                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 231                                 "size": 16
 232                             },
 233                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 234                                 {'basename': 'renamed.txt',
 235                                  'class': 'File',
 236                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 237                                  'size': 0
 238                                 }
 239                             ]}
 240                         },
 241                         'kind': 'json'
 242                     }
 243                 },
 244                 'secret_mounts': {},
 245                 'state': 'Committed',
 246                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 247                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 248                             '--eval-timeout=20', '--thread-count=0',
 249                             '--enable-reuse', "--collection-cache-size=256",
 250                             '--output-name=Output from workflow '+container_name,
 251                             '--debug', '--on-error=continue',
 252                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 253                 'name': container_name,
 254                 'container_image': '999999999999999999999999999999d3+99',
 255                 'output_name': 'Output from workflow %s' % (container_name),
 256                 'output_path': '/var/spool/cwl',
 257                 'cwd': '/var/spool/cwl',
 258                 'runtime_constraints': {
 259                     'API': True,
 260                     'vcpus': 1,
 261                     'ram': (1024+256)*1024*1024
 262                 },
 263                 'properties': stubs.git_props,
 264                 'use_existing': False,
 265                 'secret_mounts': {}
 266             }
 267
 268             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 269             stubs.api.workflows().create().execute.return_value = {
 270                 "uuid": stubs.expect_workflow_uuid,
 271             }
 272             def update_mock(**kwargs):
 273                 stubs.updated_uuid = kwargs.get('uuid')
 274                 return mock.DEFAULT
 275             stubs.api.workflows().update.side_effect = update_mock
 276             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 277                 "uuid": stubs.updated_uuid,
 278             }
 279
 280             return func(self, stubs, *args, **kwargs)
 281         return wrapped
 282     return outer_wrapper
 283
 284 class TestSubmit(unittest.TestCase):
 285
 286     def setUp(self):
 287         cwltool.process._names = set()
 288         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 289
 290     def tearDown(self):
 291         root_logger = logging.getLogger('')
 292
 293         # Remove existing RuntimeStatusLoggingHandlers if they exist
 294         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 295         root_logger.handlers = handlers
 296
 297     @mock.patch("time.sleep")
 298     @stubs()
 299     def test_submit_invalid_runner_ram(self, stubs, tm):
 300         exited = arvados_cwl.main(
 301             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 302              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 303             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 304         self.assertEqual(exited, 1)
 305
 306
 307     @stubs()
 308     def test_submit_container(self, stubs):
 309         exited = arvados_cwl.main(
 310             ["--submit", "--no-wait", "--api=containers", "--debug",
 311                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 312             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 313
 314         stubs.api.collections().create.assert_has_calls([
 315             mock.call(body=JsonDiffMatcher({
 316                 'manifest_text':
 317                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 318                 'replication_desired': None,
 319                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 320             }), ensure_unique_name=False),
 321             mock.call(body=JsonDiffMatcher({
 322                 'manifest_text':
 323                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 324                 'replication_desired': None,
 325                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 326             }), ensure_unique_name=False),
 327             ])
 328
 329         expect_container = copy.deepcopy(stubs.expect_container_spec)
 330         stubs.api.container_requests().create.assert_called_with(
 331             body=JsonDiffMatcher(expect_container))
 332         self.assertEqual(stubs.capture_stdout.getvalue(),
 333                          stubs.expect_container_request_uuid + '\n')
 334         self.assertEqual(exited, 0)
 335
 336
 337     @stubs()
 338     def test_submit_container_tool(self, stubs):
 339         # test for issue #16139
 340         exited = arvados_cwl.main(
 341             ["--submit", "--no-wait", "--api=containers", "--debug",
 342                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 343             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 344
 345         self.assertEqual(stubs.capture_stdout.getvalue(),
 346                          stubs.expect_container_request_uuid + '\n')
 347         self.assertEqual(exited, 0)
 348
 349     @stubs()
 350     def test_submit_container_no_reuse(self, stubs):
 351         exited = arvados_cwl.main(
 352             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 353                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 354             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 355
 356         expect_container = copy.deepcopy(stubs.expect_container_spec)
 357         expect_container["command"] = [
 358             'arvados-cwl-runner', '--local', '--api=containers',
 359             '--no-log-timestamps', '--disable-validate', '--disable-color',
 360             '--eval-timeout=20', '--thread-count=0',
 361             '--disable-reuse', "--collection-cache-size=256",
 362             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 363             '--debug', '--on-error=continue',
 364             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 365         expect_container["use_existing"] = False
 366
 367         stubs.api.container_requests().create.assert_called_with(
 368             body=JsonDiffMatcher(expect_container))
 369         self.assertEqual(stubs.capture_stdout.getvalue(),
 370                          stubs.expect_container_request_uuid + '\n')
 371         self.assertEqual(exited, 0)
 372
 373     @stubs(('submit_wf_no_reuse.cwl', None))
 374     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 375         exited = arvados_cwl.main(
 376             ["--submit", "--no-wait", "--api=containers", "--debug",
 377              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 378             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 379         self.assertEqual(exited, 0)
 380
 381         expect_container = copy.deepcopy(stubs.expect_container_spec)
 382         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 383         expect_container["use_existing"] = False
 384         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 385             {
 386                 "class": "WorkReuse",
 387                 "enableReuse": False,
 388             },
 389                 {
 390                     "acrContainerImage": "999999999999999999999999999999d3+99",
 391                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 392                 }
 393         ]
 394         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 395
 396         stubs.api.container_requests().create.assert_called_with(
 397             body=JsonDiffMatcher(expect_container))
 398         self.assertEqual(stubs.capture_stdout.getvalue(),
 399                          stubs.expect_container_request_uuid + '\n')
 400
 401
 402     @stubs()
 403     def test_submit_container_on_error(self, stubs):
 404         exited = arvados_cwl.main(
 405             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 406                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 407             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 408
 409         expect_container = copy.deepcopy(stubs.expect_container_spec)
 410         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 411                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 412                                        '--eval-timeout=20', '--thread-count=0',
 413                                        '--enable-reuse', "--collection-cache-size=256",
 414                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 415                                        '--debug', '--on-error=stop',
 416                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 417
 418         stubs.api.container_requests().create.assert_called_with(
 419             body=JsonDiffMatcher(expect_container))
 420         self.assertEqual(stubs.capture_stdout.getvalue(),
 421                          stubs.expect_container_request_uuid + '\n')
 422         self.assertEqual(exited, 0)
 423
 424     @stubs()
 425     def test_submit_container_output_name(self, stubs):
 426         output_name = "test_output_name"
 427
 428         exited = arvados_cwl.main(
 429             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 430                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 431             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 432
 433         expect_container = copy.deepcopy(stubs.expect_container_spec)
 434         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 435                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 436                                        '--eval-timeout=20', '--thread-count=0',
 437                                        '--enable-reuse', "--collection-cache-size=256",
 438                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 439                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 440         expect_container["output_name"] = output_name
 441
 442         stubs.api.container_requests().create.assert_called_with(
 443             body=JsonDiffMatcher(expect_container))
 444         self.assertEqual(stubs.capture_stdout.getvalue(),
 445                          stubs.expect_container_request_uuid + '\n')
 446         self.assertEqual(exited, 0)
 447
 448     @stubs()
 449     def test_submit_storage_classes(self, stubs):
 450         exited = arvados_cwl.main(
 451             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 452                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 453             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 454
 455         expect_container = copy.deepcopy(stubs.expect_container_spec)
 456         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 457                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 458                                        '--eval-timeout=20', '--thread-count=0',
 459                                        '--enable-reuse', "--collection-cache-size=256",
 460                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 461                                        "--debug",
 462                                        "--storage-classes=foo", '--on-error=continue',
 463                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 464
 465         stubs.api.container_requests().create.assert_called_with(
 466             body=JsonDiffMatcher(expect_container))
 467         self.assertEqual(stubs.capture_stdout.getvalue(),
 468                          stubs.expect_container_request_uuid + '\n')
 469         self.assertEqual(exited, 0)
 470
 471     @stubs()
 472     def test_submit_multiple_storage_classes(self, stubs):
 473         exited = arvados_cwl.main(
 474             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 475                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 477
 478         expect_container = copy.deepcopy(stubs.expect_container_spec)
 479         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 480                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 481                                        '--eval-timeout=20', '--thread-count=0',
 482                                        '--enable-reuse', "--collection-cache-size=256",
 483                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 484                                        "--debug",
 485                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 486                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 487
 488         stubs.api.container_requests().create.assert_called_with(
 489             body=JsonDiffMatcher(expect_container))
 490         self.assertEqual(stubs.capture_stdout.getvalue(),
 491                          stubs.expect_container_request_uuid + '\n')
 492         self.assertEqual(exited, 0)
 493
 494     @mock.patch("cwltool.task_queue.TaskQueue")
 495     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 496     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 497     @stubs()
 498     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 499         final_output_c = arvados.collection.Collection()
 500         make_output.return_value = ({},final_output_c)
 501
 502         def set_final_output(job_order, output_callback, runtimeContext):
 503             output_callback({"out": "zzzzz"}, "success")
 504             return []
 505         job.side_effect = set_final_output
 506
 507         exited = arvados_cwl.main(
 508             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 509                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 510             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 511
 512         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 513         self.assertEqual(exited, 0)
 514
 515     @mock.patch("cwltool.task_queue.TaskQueue")
 516     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 517     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 518     @stubs()
 519     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 520         final_output_c = arvados.collection.Collection()
 521         make_output.return_value = ({},final_output_c)
 522         stubs.api.config().get.return_value = {"default": {"Default": True}}
 523
 524         def set_final_output(job_order, output_callback, runtimeContext):
 525             output_callback({"out": "zzzzz"}, "success")
 526             return []
 527         job.side_effect = set_final_output
 528
 529         exited = arvados_cwl.main(
 530             ["--debug", "--local", "--disable-git",
 531                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 532             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 533
 534         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 535         self.assertEqual(exited, 0)
 536
 537     @mock.patch("cwltool.task_queue.TaskQueue")
 538     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 539     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 540     @stubs()
 541     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 542         final_output_c = arvados.collection.Collection()
 543         make_output.return_value = ({},final_output_c)
 544
 545         def set_final_output(job_order, output_callback, runtimeContext):
 546             output_callback({"out": "zzzzz"}, "success")
 547             return []
 548         job.side_effect = set_final_output
 549
 550         exited = arvados_cwl.main(
 551             ["--debug", "--local", "--disable-git",
 552                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 553             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 554
 555         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 556         self.assertEqual(exited, 0)
 557
 558     @stubs()
 559     def test_submit_container_output_ttl(self, stubs):
 560         exited = arvados_cwl.main(
 561             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 562                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 563             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 564
 565         expect_container = copy.deepcopy(stubs.expect_container_spec)
 566         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 567                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 568                                        '--eval-timeout=20', '--thread-count=0',
 569                                        '--enable-reuse', "--collection-cache-size=256",
 570                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 571                                        '--debug',
 572                                        '--on-error=continue',
 573                                        "--intermediate-output-ttl=3600",
 574                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 575
 576         stubs.api.container_requests().create.assert_called_with(
 577             body=JsonDiffMatcher(expect_container))
 578         self.assertEqual(stubs.capture_stdout.getvalue(),
 579                          stubs.expect_container_request_uuid + '\n')
 580         self.assertEqual(exited, 0)
 581
 582     @stubs()
 583     def test_submit_container_trash_intermediate(self, stubs):
 584         exited = arvados_cwl.main(
 585             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 586                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 587             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 588
 589
 590         expect_container = copy.deepcopy(stubs.expect_container_spec)
 591         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 592                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 593                                        '--eval-timeout=20', '--thread-count=0',
 594                                        '--enable-reuse', "--collection-cache-size=256",
 595                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 596                                        '--debug', '--on-error=continue',
 597                                        "--trash-intermediate",
 598                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 599
 600         stubs.api.container_requests().create.assert_called_with(
 601             body=JsonDiffMatcher(expect_container))
 602         self.assertEqual(stubs.capture_stdout.getvalue(),
 603                          stubs.expect_container_request_uuid + '\n')
 604         self.assertEqual(exited, 0)
 605
 606     @stubs()
 607     def test_submit_container_output_tags(self, stubs):
 608         output_tags = "tag0,tag1,tag2"
 609
 610         exited = arvados_cwl.main(
 611             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 612                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 613             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 614
 615         expect_container = copy.deepcopy(stubs.expect_container_spec)
 616         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 617                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 618                                        '--eval-timeout=20', '--thread-count=0',
 619                                        '--enable-reuse', "--collection-cache-size=256",
 620                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 621                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 622                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 623
 624         stubs.api.container_requests().create.assert_called_with(
 625             body=JsonDiffMatcher(expect_container))
 626         self.assertEqual(stubs.capture_stdout.getvalue(),
 627                          stubs.expect_container_request_uuid + '\n')
 628         self.assertEqual(exited, 0)
 629
 630     @stubs()
 631     def test_submit_container_runner_ram(self, stubs):
 632         exited = arvados_cwl.main(
 633             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 634                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 635             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 636
 637         expect_container = copy.deepcopy(stubs.expect_container_spec)
 638         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 639         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 640                 {
 641                     "acrContainerImage": "999999999999999999999999999999d3+99",
 642                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 643                     "ramMin": 2048
 644                 }
 645         ]
 646
 647         stubs.api.container_requests().create.assert_called_with(
 648             body=JsonDiffMatcher(expect_container))
 649         self.assertEqual(stubs.capture_stdout.getvalue(),
 650                          stubs.expect_container_request_uuid + '\n')
 651         self.assertEqual(exited, 0)
 652
 653     @mock.patch("arvados.collection.CollectionReader")
 654     @mock.patch("time.sleep")
 655     @stubs()
 656     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 657         collectionReader().exists.return_value = True
 658         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 659         exited = arvados_cwl.main(
 660             ["--submit", "--no-wait", "--api=containers", "--debug",
 661              "tests/wf/submit_keepref_wf.cwl"],
 662             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 663         self.assertEqual(exited, 0)
 664
 665     @mock.patch("arvados.collection.CollectionReader")
 666     @mock.patch("time.sleep")
 667     @stubs()
 668     def test_submit_keepref(self, stubs, tm, reader):
 669         with open("tests/wf/expect_arvworkflow.cwl") as f:
 670             reader().open().__enter__().read.return_value = f.read()
 671
 672         exited = arvados_cwl.main(
 673             ["--submit", "--no-wait", "--api=containers", "--debug",
 674              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 675             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 676
 677         expect_container = {
 678             'priority': 500,
 679             'mounts': {
 680                 '/var/spool/cwl': {
 681                     'writable': True,
 682                     'kind': 'collection'
 683                 },
 684                 'stdout': {
 685                     'path': '/var/spool/cwl/cwl.output.json',
 686                     'kind': 'file'
 687                 },
 688                 '/var/lib/cwl/workflow': {
 689                     'portable_data_hash': '99999999999999999999999999999994+99',
 690                     'kind': 'collection'
 691                 },
 692                 '/var/lib/cwl/cwl.input.json': {
 693                     'content': {
 694                         'x': 'XxX'
 695                     },
 696                     'kind': 'json'
 697                 }
 698             }, 'state': 'Committed',
 699             'output_path': '/var/spool/cwl',
 700             'name': 'expect_arvworkflow.cwl#main',
 701             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 702             'container_image': '999999999999999999999999999999d3+99',
 703             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 704                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 705                         '--eval-timeout=20', '--thread-count=0',
 706                         '--enable-reuse', "--collection-cache-size=256",
 707                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 708                         '--debug', '--on-error=continue',
 709                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 710             'cwd': '/var/spool/cwl',
 711             'runtime_constraints': {
 712                 'API': True,
 713                 'vcpus': 1,
 714                 'ram': 1342177280
 715             },
 716             'use_existing': False,
 717             'properties': {},
 718             'secret_mounts': {}
 719         }
 720
 721         stubs.api.container_requests().create.assert_called_with(
 722             body=JsonDiffMatcher(expect_container))
 723         self.assertEqual(stubs.capture_stdout.getvalue(),
 724                          stubs.expect_container_request_uuid + '\n')
 725         self.assertEqual(exited, 0)
 726
 727     @mock.patch("time.sleep")
 728     @stubs()
 729     def test_submit_arvworkflow(self, stubs, tm):
 730         with open("tests/wf/expect_arvworkflow.cwl") as f:
 731             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 732
 733         exited = arvados_cwl.main(
 734             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 735              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 736             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 737
 738         expect_container = {
 739             'priority': 500,
 740             'mounts': {
 741                 '/var/spool/cwl': {
 742                     'writable': True,
 743                     'kind': 'collection'
 744                 },
 745                 'stdout': {
 746                     'path': '/var/spool/cwl/cwl.output.json',
 747                     'kind': 'file'
 748                 },
 749                 '/var/lib/cwl/workflow.json': {
 750                     'kind': 'json',
 751                     'content': {
 752                         'cwlVersion': 'v1.0',
 753                         'label': 'a test workflow',
 754                         '$graph': [
 755                             {
 756                                 'id': '#main',
 757                                 'inputs': [
 758                                     {'type': 'string', 'id': '#main/x'}
 759                                 ],
 760                                 'steps': [
 761                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 762                                      'run': '#submit_tool.cwl',
 763                                      'id': '#main/step1',
 764                                      'out': []}
 765                                 ],
 766                                 'class': 'Workflow',
 767                                 'outputs': []
 768                             },
 769                             {
 770                                 'inputs': [
 771                                     {
 772                                         'inputBinding': {'position': 1},
 773                                         'type': 'string',
 774                                         'id': '#submit_tool.cwl/x'}
 775                                 ],
 776                                 'requirements': [
 777                                     {
 778                                         'dockerPull': 'debian:buster-slim',
 779                                         'class': 'DockerRequirement'
 780                                     }
 781                                 ],
 782                                 'id': '#submit_tool.cwl',
 783                                 'outputs': [],
 784                                 'baseCommand': 'cat',
 785                                 'class': 'CommandLineTool'
 786                             }
 787                         ]
 788                     }
 789                 },
 790                 '/var/lib/cwl/cwl.input.json': {
 791                     'content': {
 792                         'x': 'XxX'
 793                     },
 794                     'kind': 'json'
 795                 }
 796             }, 'state': 'Committed',
 797             'output_path': '/var/spool/cwl',
 798             'name': 'a test workflow',
 799             'container_image': "999999999999999999999999999999d3+99",
 800             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 801                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 802                         '--eval-timeout=20', '--thread-count=0',
 803                         '--enable-reuse', "--collection-cache-size=256",
 804                         "--output-name=Output from workflow a test workflow",
 805                         '--debug', '--on-error=continue',
 806                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 807             'output_name': 'Output from workflow a test workflow',
 808             'cwd': '/var/spool/cwl',
 809             'runtime_constraints': {
 810                 'API': True,
 811                 'vcpus': 1,
 812                 'ram': 1342177280
 813             },
 814             'use_existing': False,
 815             'properties': {
 816                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 817             },
 818             'secret_mounts': {}
 819         }
 820
 821         stubs.api.container_requests().create.assert_called_with(
 822             body=JsonDiffMatcher(expect_container))
 823         self.assertEqual(stubs.capture_stdout.getvalue(),
 824                          stubs.expect_container_request_uuid + '\n')
 825         self.assertEqual(exited, 0)
 826
 827     @stubs(('hello container 123', 'submit_wf.cwl'))
 828     def test_submit_container_name(self, stubs):
 829         exited = arvados_cwl.main(
 830             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 831                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 832             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 833
 834         expect_container = copy.deepcopy(stubs.expect_container_spec)
 835
 836         stubs.api.container_requests().create.assert_called_with(
 837             body=JsonDiffMatcher(expect_container))
 838         self.assertEqual(stubs.capture_stdout.getvalue(),
 839                          stubs.expect_container_request_uuid + '\n')
 840         self.assertEqual(exited, 0)
 841
 842     @stubs()
 843     def test_submit_missing_input(self, stubs):
 844         exited = arvados_cwl.main(
 845             ["--submit", "--no-wait", "--api=containers", "--debug",
 846              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 847             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 848         self.assertEqual(exited, 0)
 849
 850         exited = arvados_cwl.main(
 851             ["--submit", "--no-wait", "--api=containers", "--debug",
 852              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 853             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 854         self.assertEqual(exited, 1)
 855
 856     @stubs()
 857     def test_submit_container_project(self, stubs):
 858         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 859         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 860         exited = arvados_cwl.main(
 861             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 862                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 863             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 864
 865         expect_container = copy.deepcopy(stubs.expect_container_spec)
 866         expect_container["owner_uuid"] = project_uuid
 867         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 868                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 869                                        "--eval-timeout=20", "--thread-count=0",
 870                                        '--enable-reuse', "--collection-cache-size=256",
 871                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 872                                        '--debug',
 873                                        '--on-error=continue',
 874                                        '--project-uuid='+project_uuid,
 875                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 876
 877         stubs.api.container_requests().create.assert_called_with(
 878             body=JsonDiffMatcher(expect_container))
 879         self.assertEqual(stubs.capture_stdout.getvalue(),
 880                          stubs.expect_container_request_uuid + '\n')
 881         self.assertEqual(exited, 0)
 882
 883     @stubs()
 884     def test_submit_container_eval_timeout(self, stubs):
 885         exited = arvados_cwl.main(
 886             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 887                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 888             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 889
 890         expect_container = copy.deepcopy(stubs.expect_container_spec)
 891         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 892                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 893                                        '--eval-timeout=60.0', '--thread-count=0',
 894                                        '--enable-reuse', "--collection-cache-size=256",
 895                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 896                                        '--debug', '--on-error=continue',
 897                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 898
 899         stubs.api.container_requests().create.assert_called_with(
 900             body=JsonDiffMatcher(expect_container))
 901         self.assertEqual(stubs.capture_stdout.getvalue(),
 902                          stubs.expect_container_request_uuid + '\n')
 903         self.assertEqual(exited, 0)
 904
 905     @stubs()
 906     def test_submit_container_collection_cache(self, stubs):
 907         exited = arvados_cwl.main(
 908             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 909                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 910             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 911
 912         expect_container = copy.deepcopy(stubs.expect_container_spec)
 913         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 914                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 915                                        '--eval-timeout=20', '--thread-count=0',
 916                                        '--enable-reuse', "--collection-cache-size=500",
 917                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 918                                        '--debug', '--on-error=continue',
 919                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 920         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 921
 922         stubs.api.container_requests().create.assert_called_with(
 923             body=JsonDiffMatcher(expect_container))
 924         self.assertEqual(stubs.capture_stdout.getvalue(),
 925                          stubs.expect_container_request_uuid + '\n')
 926         self.assertEqual(exited, 0)
 927
 928     @stubs()
 929     def test_submit_container_thread_count(self, stubs):
 930         exited = arvados_cwl.main(
 931             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 932                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 933             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 934
 935         expect_container = copy.deepcopy(stubs.expect_container_spec)
 936         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 937                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 938                                        '--eval-timeout=20', '--thread-count=20',
 939                                        '--enable-reuse', "--collection-cache-size=256",
 940                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 941                                        '--debug', '--on-error=continue',
 942                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 943
 944         stubs.api.container_requests().create.assert_called_with(
 945             body=JsonDiffMatcher(expect_container))
 946         self.assertEqual(stubs.capture_stdout.getvalue(),
 947                          stubs.expect_container_request_uuid + '\n')
 948         self.assertEqual(exited, 0)
 949
 950     @stubs()
 951     def test_submit_container_runner_image(self, stubs):
 952         exited = arvados_cwl.main(
 953             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 954                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 955             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 956
 957         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 958         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 959                 {
 960                     "acrContainerImage": "999999999999999999999999999999d5+99",
 961                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 962                 }
 963         ]
 964
 965         expect_container = copy.deepcopy(stubs.expect_container_spec)
 966         stubs.api.container_requests().create.assert_called_with(
 967             body=JsonDiffMatcher(expect_container))
 968         self.assertEqual(stubs.capture_stdout.getvalue(),
 969                          stubs.expect_container_request_uuid + '\n')
 970         self.assertEqual(exited, 0)
 971
 972     @stubs()
 973     def test_submit_priority(self, stubs):
 974         exited = arvados_cwl.main(
 975             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 976                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 977             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 978
 979         stubs.expect_container_spec["priority"] = 669
 980
 981         expect_container = copy.deepcopy(stubs.expect_container_spec)
 982         stubs.api.container_requests().create.assert_called_with(
 983             body=JsonDiffMatcher(expect_container))
 984         self.assertEqual(stubs.capture_stdout.getvalue(),
 985                          stubs.expect_container_request_uuid + '\n')
 986         self.assertEqual(exited, 0)
 987
 988     @stubs(('submit_wf_runner_resources.cwl', None))
 989     def test_submit_wf_runner_resources(self, stubs):
 990         exited = arvados_cwl.main(
 991             ["--submit", "--no-wait", "--api=containers", "--debug",
 992                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 993             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 994
 995         expect_container = copy.deepcopy(stubs.expect_container_spec)
 996         expect_container["runtime_constraints"] = {
 997             "API": True,
 998             "vcpus": 2,
 999             "ram": (2000+512) * 2**20
1000         }
1001         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1002             {
1003                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1004                 "acrContainerImage": "999999999999999999999999999999d3+99",
1005                 "coresMin": 2,
1006                 "ramMin": 2000,
1007                 "keep_cache": 512
1008             }
1009         ]
1010         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1011         #    "arv": "http://arvados.org/cwl#",
1012         #}
1013         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1014         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1015
1016         stubs.api.container_requests().create.assert_called_with(
1017             body=JsonDiffMatcher(expect_container))
1018         self.assertEqual(stubs.capture_stdout.getvalue(),
1019                          stubs.expect_container_request_uuid + '\n')
1020         self.assertEqual(exited, 0)
1021
1022     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1023     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1024     @mock.patch("arvados.api")
1025     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1026         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1027
1028         arvrunner = mock.MagicMock()
1029         arvrunner.project_uuid = ""
1030         api.return_value = mock.MagicMock()
1031         arvrunner.api = api.return_value
1032         arvrunner.runtimeContext.match_local_docker = False
1033         arvrunner.api.links().list().execute.side_effect = itertools.cycle([
1034             {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
1035                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1036                         "link_class": "docker_image_repo+tag",
1037                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1038                         "owner_uuid": "",
1039                         "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
1040                         "properties": {"image_timestamp": ""}}]},
1041             {"items": []},
1042             {"items": []},
1043             {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
1044                         "head_uuid": "",
1045                         "link_class": "docker_image_hash",
1046                         "name": "123456",
1047                         "owner_uuid": "",
1048                         "uuid": "zzzzz-o0j2j-arvadosjobshash",
1049                         "properties": {"image_timestamp": ""}}]},
1050             {"items": []},
1051             {"items": []},
1052         ])
1053         find_one_image_hash.return_value = "123456"
1054
1055         arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
1056             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1057                         "owner_uuid": "",
1058                         "manifest_text": "",
1059                         "created_at": "2023-08-25T12:34:55.012345Z",
1060                         "properties": {}}]},
1061             {"items": []},
1062             {"items": []},
1063         ])
1064         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1065         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1066                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1067
1068         self.assertEqual("9999999999999999999999999999999b+99",
1069                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1070
1071
1072     @stubs()
1073     def test_submit_secrets(self, stubs):
1074         exited = arvados_cwl.main(
1075             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1076                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1077             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1078
1079         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1080         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1081
1082         expect_container = {
1083             "command": [
1084                 "arvados-cwl-runner",
1085                 "--local",
1086                 "--api=containers",
1087                 "--no-log-timestamps",
1088                 "--disable-validate",
1089                 "--disable-color",
1090                 "--eval-timeout=20",
1091                 '--thread-count=0',
1092                 "--enable-reuse",
1093                 "--collection-cache-size=256",
1094                 "--output-name=Output from workflow secret_wf.cwl",
1095                 "--debug",
1096                 "--on-error=continue",
1097                 "/var/lib/cwl/workflow.json#main",
1098                 "/var/lib/cwl/cwl.input.json"
1099             ],
1100             "container_image": "999999999999999999999999999999d3+99",
1101             "cwd": "/var/spool/cwl",
1102             "mounts": {
1103                 "/var/lib/cwl/cwl.input.json": {
1104                     "content": {
1105                         "pw": {
1106                             "$include": "/secrets/s0"
1107                         }
1108                     },
1109                     "kind": "json"
1110                 },
1111                 "/var/lib/cwl/workflow.json": {
1112                     "content": {
1113                         "$graph": [
1114                             {
1115                                 "class": "Workflow",
1116                                 "hints": [
1117                                     {
1118                                     "class": "DockerRequirement",
1119                                     "dockerPull": "debian:buster-slim",
1120                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1121                                     },
1122                                     {
1123                                         "class": "http://commonwl.org/cwltool#Secrets",
1124                                         "secrets": [
1125                                             "#main/pw"
1126                                         ]
1127                                     },
1128                                     {
1129                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1130                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1131                                     }
1132                                 ],
1133                                 "id": "#main",
1134                                 "inputs": [
1135                                     {
1136                                         "id": "#main/pw",
1137                                         "type": "string"
1138                                     }
1139                                 ],
1140                                 "outputs": [
1141                                     {
1142                                         "id": "#main/out",
1143                                         "outputSource": "#main/step/out",
1144                                         "type": "File"
1145                                     }
1146                                 ],
1147                                 "requirements": [
1148                                     {
1149                                         "class": "SubworkflowFeatureRequirement"
1150                                     }
1151                                 ],
1152                                 "steps": [
1153                                     {
1154                                         "id": "#main/secret_wf.cwl",
1155                                         "in": [
1156                                             {
1157                                                 "id": "#main/step/pw",
1158                                                 "source": "#main/pw"
1159                                             }
1160                                         ],
1161                                         "label": "secret_wf.cwl",
1162                                         "out": [
1163                                             {"id": "#main/step/out"}
1164                                         ],
1165                                         "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
1166                                     }
1167                                 ]
1168                             }
1169                         ],
1170                         "cwlVersion": "v1.2"
1171                     },
1172                     "kind": "json"
1173                 },
1174                 "/var/spool/cwl": {
1175                     "kind": "collection",
1176                     "writable": True
1177                 },
1178                 "stdout": {
1179                     "kind": "file",
1180                     "path": "/var/spool/cwl/cwl.output.json"
1181                 }
1182             },
1183             "name": "secret_wf.cwl",
1184             "output_name": "Output from workflow secret_wf.cwl",
1185             "output_path": "/var/spool/cwl",
1186             "priority": 500,
1187             "properties": {},
1188             "runtime_constraints": {
1189                 "API": True,
1190                 "ram": 1342177280,
1191                 "vcpus": 1
1192             },
1193             "secret_mounts": {
1194                 "/secrets/s0": {
1195                     "content": "blorp",
1196                     "kind": "text"
1197                 }
1198             },
1199             "state": "Committed",
1200             "use_existing": False
1201         }
1202
1203         stubs.api.container_requests().create.assert_called_with(
1204             body=JsonDiffMatcher(expect_container))
1205         self.assertEqual(stubs.capture_stdout.getvalue(),
1206                          stubs.expect_container_request_uuid + '\n')
1207         self.assertEqual(exited, 0)
1208
1209     @stubs()
1210     def test_submit_request_uuid(self, stubs):
1211         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1212         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1213
1214         stubs.api.container_requests().update().execute.return_value = {
1215             "uuid": stubs.expect_container_request_uuid,
1216             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1217             "state": "Queued"
1218         }
1219
1220         exited = arvados_cwl.main(
1221             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1222                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1223             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1224
1225         stubs.api.container_requests().update.assert_called_with(
1226             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1227         self.assertEqual(stubs.capture_stdout.getvalue(),
1228                          stubs.expect_container_request_uuid + '\n')
1229         self.assertEqual(exited, 0)
1230
1231     @stubs()
1232     def test_submit_container_cluster_id(self, stubs):
1233         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1234
1235         exited = arvados_cwl.main(
1236             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1237                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1238             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1239
1240         expect_container = copy.deepcopy(stubs.expect_container_spec)
1241
1242         stubs.api.container_requests().create.assert_called_with(
1243             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1244         self.assertEqual(stubs.capture_stdout.getvalue(),
1245                          stubs.expect_container_request_uuid + '\n')
1246         self.assertEqual(exited, 0)
1247
1248     @stubs()
1249     def test_submit_validate_cluster_id(self, stubs):
1250         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1251         exited = arvados_cwl.main(
1252             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1253              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1254             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1255         self.assertEqual(exited, 1)
1256
1257     @stubs()
1258     def test_submit_validate_project_uuid(self, stubs):
1259         # Fails with bad cluster prefix
1260         exited = arvados_cwl.main(
1261             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1262              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1263             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1264         self.assertEqual(exited, 1)
1265
1266         # Project lookup fails
1267         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1268         exited = arvados_cwl.main(
1269             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1270              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1271             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1272         self.assertEqual(exited, 1)
1273
1274         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1275         exited = arvados_cwl.main(
1276             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1277              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1278             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1279         self.assertEqual(exited, 0)
1280
1281
1282     @mock.patch("arvados.collection.CollectionReader")
1283     @stubs()
1284     def test_submit_uuid_inputs(self, stubs, collectionReader):
1285         collectionReader().exists.return_value = True
1286         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1287         def list_side_effect(**kwargs):
1288             m = mock.MagicMock()
1289             if "count" in kwargs:
1290                 m.execute.return_value = {"items": [
1291                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1292                 ]}
1293             else:
1294                 m.execute.return_value = {"items": []}
1295             return m
1296         stubs.api.collections().list.side_effect = list_side_effect
1297         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1298
1299         exited = arvados_cwl.main(
1300             ["--submit", "--no-wait", "--api=containers", "--debug",
1301                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1302             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1303
1304         expect_container = copy.deepcopy(stubs.expect_container_spec)
1305         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1306         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1307         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1308
1309         stubs.api.collections().list.assert_has_calls([
1310             mock.call(count='none',
1311                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1312                       select=['uuid', 'portable_data_hash'])])
1313         stubs.api.container_requests().create.assert_called_with(
1314             body=JsonDiffMatcher(expect_container))
1315         self.assertEqual(stubs.capture_stdout.getvalue(),
1316                          stubs.expect_container_request_uuid + '\n')
1317         self.assertEqual(exited, 0)
1318
1319     @stubs()
1320     def test_submit_mismatched_uuid_inputs(self, stubs):
1321         def list_side_effect(**kwargs):
1322             m = mock.MagicMock()
1323             if "count" in kwargs:
1324                 m.execute.return_value = {"items": [
1325                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1326                 ]}
1327             else:
1328                 m.execute.return_value = {"items": []}
1329             return m
1330         stubs.api.collections().list.side_effect = list_side_effect
1331
1332         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1333             capture_stderr = StringIO()
1334             cwltool_logger = logging.getLogger('cwltool')
1335             stderr_logger = logging.StreamHandler(capture_stderr)
1336             cwltool_logger.addHandler(stderr_logger)
1337
1338             try:
1339                 exited = arvados_cwl.main(
1340                     ["--submit", "--no-wait", "--api=containers", "--debug",
1341                         "tests/wf/submit_wf.cwl", infile],
1342                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1343
1344                 self.assertEqual(exited, 1)
1345                 self.assertRegex(
1346                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1347                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1348             finally:
1349                 cwltool_logger.removeHandler(stderr_logger)
1350
1351     @mock.patch("arvados.collection.CollectionReader")
1352     @stubs()
1353     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1354         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1355         capture_stderr = StringIO()
1356
1357         cwltool_logger = logging.getLogger('cwltool')
1358         stderr_logger = logging.StreamHandler(capture_stderr)
1359         cwltool_logger.addHandler(stderr_logger)
1360
1361         exited = arvados_cwl.main(
1362             ["--submit", "--no-wait", "--api=containers", "--debug",
1363                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1364             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1365
1366         try:
1367             self.assertEqual(exited, 1)
1368             self.assertRegex(
1369                 capture_stderr.getvalue(),
1370                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1371         finally:
1372             cwltool_logger.removeHandler(stderr_logger)
1373
1374     @stubs(('submit_wf_process_properties.cwl', None))
1375     def test_submit_set_process_properties(self, stubs):
1376         exited = arvados_cwl.main(
1377             ["--submit", "--no-wait", "--api=containers", "--debug",
1378                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1379             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1380
1381         expect_container = copy.deepcopy(stubs.expect_container_spec)
1382
1383         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1384             {
1385                 "class": "http://arvados.org/cwl#ProcessProperties",
1386                 "processProperties": [
1387                     {"propertyName": "baz",
1388                      "propertyValue": "$(inputs.x.basename)"},
1389                     {"propertyName": "foo",
1390                      "propertyValue": "bar"},
1391                     {"propertyName": "quux",
1392                      "propertyValue": {
1393                          "q1": 1,
1394                          "q2": 2
1395                      }
1396                     }
1397                 ],
1398             },
1399             {
1400                 "acrContainerImage": "999999999999999999999999999999d3+99",
1401                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1402             }
1403         ]
1404         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1405         #    "arv": "http://arvados.org/cwl#"
1406         #}
1407
1408         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1409
1410         expect_container["properties"].update({
1411             "baz": "blorp.txt",
1412             "foo": "bar",
1413             "quux": {
1414                 "q1": 1,
1415                 "q2": 2
1416             }
1417         })
1418
1419         stubs.api.container_requests().create.assert_called_with(
1420             body=JsonDiffMatcher(expect_container))
1421         self.assertEqual(stubs.capture_stdout.getvalue(),
1422                          stubs.expect_container_request_uuid + '\n')
1423         self.assertEqual(exited, 0)
1424
1425
1426     @stubs()
1427     def test_submit_enable_preemptible(self, stubs):
1428         exited = arvados_cwl.main(
1429             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1430                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1431             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1432
1433         expect_container = copy.deepcopy(stubs.expect_container_spec)
1434         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1435                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1436                                        '--eval-timeout=20', '--thread-count=0',
1437                                        '--enable-reuse', "--collection-cache-size=256",
1438                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1439                                        '--debug', '--on-error=continue',
1440                                        '--enable-preemptible',
1441                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1442
1443         stubs.api.container_requests().create.assert_called_with(
1444             body=JsonDiffMatcher(expect_container))
1445         self.assertEqual(stubs.capture_stdout.getvalue(),
1446                          stubs.expect_container_request_uuid + '\n')
1447         self.assertEqual(exited, 0)
1448
1449     @stubs()
1450     def test_submit_disable_preemptible(self, stubs):
1451         exited = arvados_cwl.main(
1452             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1453                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1454             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1455
1456         expect_container = copy.deepcopy(stubs.expect_container_spec)
1457         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1458                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1459                                        '--eval-timeout=20', '--thread-count=0',
1460                                        '--enable-reuse', "--collection-cache-size=256",
1461                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1462                                        '--debug', '--on-error=continue',
1463                                        '--disable-preemptible',
1464                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1465
1466         stubs.api.container_requests().create.assert_called_with(
1467             body=JsonDiffMatcher(expect_container))
1468         self.assertEqual(stubs.capture_stdout.getvalue(),
1469                          stubs.expect_container_request_uuid + '\n')
1470         self.assertEqual(exited, 0)
1471
1472     @stubs()
1473     def test_submit_container_prefer_cached_downloads(self, stubs):
1474         exited = arvados_cwl.main(
1475             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1476                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1477             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1478
1479         expect_container = copy.deepcopy(stubs.expect_container_spec)
1480         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1481                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1482                                        '--eval-timeout=20', '--thread-count=0',
1483                                        '--enable-reuse', "--collection-cache-size=256",
1484                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1485                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1486                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1487
1488         stubs.api.container_requests().create.assert_called_with(
1489             body=JsonDiffMatcher(expect_container))
1490         self.assertEqual(stubs.capture_stdout.getvalue(),
1491                          stubs.expect_container_request_uuid + '\n')
1492         self.assertEqual(exited, 0)
1493
1494     @stubs()
1495     def test_submit_container_varying_url_params(self, stubs):
1496         exited = arvados_cwl.main(
1497             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1498                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1499             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1500
1501         expect_container = copy.deepcopy(stubs.expect_container_spec)
1502         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1503                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1504                                        '--eval-timeout=20', '--thread-count=0',
1505                                        '--enable-reuse', "--collection-cache-size=256",
1506                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1507                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1508                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1509
1510         stubs.api.container_requests().create.assert_called_with(
1511             body=JsonDiffMatcher(expect_container))
1512         self.assertEqual(stubs.capture_stdout.getvalue(),
1513                          stubs.expect_container_request_uuid + '\n')
1514         self.assertEqual(exited, 0)
1515
1516
1517 class TestCreateWorkflow(unittest.TestCase):
1518     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1519     expect_workflow = StripYAMLComments(
1520         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1521     expect_workflow_altname = StripYAMLComments(
1522         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1523
1524     def setUp(self):
1525         cwltool.process._names = set()
1526         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1527
1528     def tearDown(self):
1529         root_logger = logging.getLogger('')
1530
1531         # Remove existing RuntimeStatusLoggingHandlers if they exist
1532         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1533         root_logger.handlers = handlers
1534
1535     @stubs()
1536     def test_create(self, stubs):
1537         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1538         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1539
1540         exited = arvados_cwl.main(
1541             ["--create-workflow", "--debug",
1542              "--api=containers",
1543              "--project-uuid", project_uuid,
1544              "--disable-git",
1545              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1546             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1547
1548         stubs.api.pipeline_templates().create.refute_called()
1549         stubs.api.container_requests().create.refute_called()
1550
1551         body = {
1552             "workflow": {
1553                 "owner_uuid": project_uuid,
1554                 "name": "submit_wf.cwl",
1555                 "description": "",
1556                 "definition": self.expect_workflow,
1557             }
1558         }
1559         stubs.api.workflows().create.assert_called_with(
1560             body=JsonDiffMatcher(body))
1561
1562         self.assertEqual(stubs.capture_stdout.getvalue(),
1563                          stubs.expect_workflow_uuid + '\n')
1564         self.assertEqual(exited, 0)
1565
1566     @stubs()
1567     def test_create_name(self, stubs):
1568         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1569         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1570
1571         exited = arvados_cwl.main(
1572             ["--create-workflow", "--debug",
1573              "--api=containers",
1574              "--project-uuid", project_uuid,
1575              "--name", "testing 123",
1576              "--disable-git",
1577              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1578             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1579
1580         stubs.api.pipeline_templates().create.refute_called()
1581         stubs.api.container_requests().create.refute_called()
1582
1583         body = {
1584             "workflow": {
1585                 "owner_uuid": project_uuid,
1586                 "name": "testing 123",
1587                 "description": "",
1588                 "definition": self.expect_workflow_altname,
1589             }
1590         }
1591         stubs.api.workflows().create.assert_called_with(
1592             body=JsonDiffMatcher(body))
1593
1594         self.assertEqual(stubs.capture_stdout.getvalue(),
1595                          stubs.expect_workflow_uuid + '\n')
1596         self.assertEqual(exited, 0)
1597
1598
1599     @stubs()
1600     def test_update(self, stubs):
1601         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1602         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1603
1604         exited = arvados_cwl.main(
1605             ["--update-workflow", self.existing_workflow_uuid,
1606              "--debug",
1607              "--disable-git",
1608              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1609             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1610
1611         body = {
1612             "workflow": {
1613                 "name": "submit_wf.cwl",
1614                 "description": "",
1615                 "definition": self.expect_workflow,
1616                 "owner_uuid": project_uuid
1617             }
1618         }
1619         stubs.api.workflows().update.assert_called_with(
1620             uuid=self.existing_workflow_uuid,
1621             body=JsonDiffMatcher(body))
1622         self.assertEqual(stubs.capture_stdout.getvalue(),
1623                          self.existing_workflow_uuid + '\n')
1624         self.assertEqual(exited, 0)
1625
1626
1627     @stubs()
1628     def test_update_name(self, stubs):
1629         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1630         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1631
1632         exited = arvados_cwl.main(
1633             ["--update-workflow", self.existing_workflow_uuid,
1634              "--debug", "--name", "testing 123",
1635              "--disable-git",
1636              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1637             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1638
1639         body = {
1640             "workflow": {
1641                 "name": "testing 123",
1642                 "description": "",
1643                 "definition": self.expect_workflow_altname,
1644                 "owner_uuid": project_uuid
1645             }
1646         }
1647         stubs.api.workflows().update.assert_called_with(
1648             uuid=self.existing_workflow_uuid,
1649             body=JsonDiffMatcher(body))
1650         self.assertEqual(stubs.capture_stdout.getvalue(),
1651                          self.existing_workflow_uuid + '\n')
1652         self.assertEqual(exited, 0)
1653
1654     @stubs()
1655     def test_create_collection_per_tool(self, stubs):
1656         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1657         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1658
1659         exited = arvados_cwl.main(
1660             ["--create-workflow", "--debug",
1661              "--api=containers",
1662              "--project-uuid", project_uuid,
1663              "--disable-git",
1664              "tests/collection_per_tool/collection_per_tool.cwl"],
1665             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1666
1667         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1668         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1669
1670         body = {
1671             "workflow": {
1672                 "owner_uuid": project_uuid,
1673                 "name": "collection_per_tool.cwl",
1674                 "description": "",
1675                 "definition": expect_workflow,
1676             }
1677         }
1678         stubs.api.workflows().create.assert_called_with(
1679             body=JsonDiffMatcher(body))
1680
1681         self.assertEqual(stubs.capture_stdout.getvalue(),
1682                          stubs.expect_workflow_uuid + '\n')
1683         self.assertEqual(exited, 0)
1684
1685     @stubs()
1686     def test_create_with_imports(self, stubs):
1687         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1688         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1689
1690         exited = arvados_cwl.main(
1691             ["--create-workflow", "--debug",
1692              "--api=containers",
1693              "--project-uuid", project_uuid,
1694              "tests/wf/feddemo/feddemo.cwl"],
1695             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1696
1697         stubs.api.pipeline_templates().create.refute_called()
1698         stubs.api.container_requests().create.refute_called()
1699
1700         self.assertEqual(stubs.capture_stdout.getvalue(),
1701                          stubs.expect_workflow_uuid + '\n')
1702         self.assertEqual(exited, 0)
1703
1704     @stubs()
1705     def test_create_with_no_input(self, stubs):
1706         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1707         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1708
1709         exited = arvados_cwl.main(
1710             ["--create-workflow", "--debug",
1711              "--api=containers",
1712              "--project-uuid", project_uuid,
1713              "tests/wf/revsort/revsort.cwl"],
1714             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1715
1716         stubs.api.pipeline_templates().create.refute_called()
1717         stubs.api.container_requests().create.refute_called()
1718
1719         self.assertEqual(stubs.capture_stdout.getvalue(),
1720                          stubs.expect_workflow_uuid + '\n')
1721         self.assertEqual(exited, 0)
1722
1723     @stubs()
1724     def test_create_map(self, stubs):
1725         # test uploading a document that uses objects instead of arrays
1726         # for certain fields like inputs and requirements.
1727
1728         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1729         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1730
1731         exited = arvados_cwl.main(
1732             ["--create-workflow", "--debug",
1733              "--api=containers",
1734              "--project-uuid", project_uuid,
1735              "--disable-git",
1736              "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
1737             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1738
1739         stubs.api.pipeline_templates().create.refute_called()
1740         stubs.api.container_requests().create.refute_called()
1741
1742         expect_workflow = StripYAMLComments(
1743             open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
1744
1745         body = {
1746             "workflow": {
1747                 "owner_uuid": project_uuid,
1748                 "name": "submit_wf_map.cwl",
1749                 "description": "",
1750                 "definition": expect_workflow,
1751             }
1752         }
1753         stubs.api.workflows().create.assert_called_with(
1754             body=JsonDiffMatcher(body))
1755
1756         self.assertEqual(stubs.capture_stdout.getvalue(),
1757                          stubs.expect_workflow_uuid + '\n')
1758         self.assertEqual(exited, 0)
1759
1760
1761 class TestPrintKeepDeps(unittest.TestCase):
1762     @stubs()
1763     def test_print_keep_deps(self, stubs):
1764         # test --print-keep-deps which is used by arv-copy
1765
1766         exited = arvados_cwl.main(
1767             ["--print-keep-deps", "--debug",
1768              "tests/wf/submit_wf_map.cwl"],
1769             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1770
1771         self.assertEqual(stubs.capture_stdout.getvalue(),
1772                          '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
1773         self.assertEqual(exited, 0)