sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import io
   7 import itertools
   8 import functools
   9 import hashlib
  10 import json
  11 import logging
  12 import sys
  13 import unittest
  14 import re
  15 import os
  16 import collections
  17
  18 from io import BytesIO, StringIO
  19 from unittest import mock
  20
  21 import arvados
  22 import arvados.collection
  23 import arvados_cwl
  24 import arvados_cwl.executor
  25 import arvados_cwl.runner
  26 import arvados.keep
  27
  28 import cwltool.process
  29
  30 from .matcher import JsonDiffMatcher, StripYAMLComments
  31 from .mock_discovery import get_rootDesc
  32
  33 import ruamel.yaml
  34
  35 _rootDesc = None
  36
  37 def stubs(wfdetails=('submit_wf.cwl', None)):
  38     def outer_wrapper(func, *rest):
  39         @functools.wraps(func)
  40         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  41         @mock.patch("uuid.uuid4")
  42         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  43         @mock.patch("arvados.collection.KeepClient")
  44         @mock.patch("arvados.keep.KeepClient")
  45         @mock.patch("arvados.events.subscribe")
  46         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  47                     uuid4, determine_image_id, *args, **kwargs):
  48             class Stubs(object):
  49                 pass
  50
  51             wfname = wfdetails[0]
  52             wfpath = wfdetails[1]
  53
  54             stubs = Stubs()
  55             stubs.events = events
  56             stubs.keepdocker = keepdocker
  57
  58             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  59                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  60                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  61                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  62                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  63                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  64                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  65                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  66                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  67
  68             determine_image_id.return_value = None
  69
  70             def putstub(p, **kwargs):
  71                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  72             keep_client1().put.side_effect = putstub
  73             keep_client1.put.side_effect = putstub
  74             keep_client2().put.side_effect = putstub
  75             keep_client2.put.side_effect = putstub
  76
  77             stubs.keep_client = keep_client2
  78             stubs.docker_images = {
  79                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  80                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  81                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  82                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  83             }
  84             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  85                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  86             stubs.keepdocker.side_effect = kd
  87
  88             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  89             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  90
  91             stubs.capture_stdout = StringIO()
  92
  93             stubs.api = mock.MagicMock()
  94             stubs.api._rootDesc = get_rootDesc()
  95             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  96             stubs.api._rootDesc["revision"] = "20210628"
  97
  98             stubs.api.users().current().execute.return_value = {
  99                 "uuid": stubs.fake_user_uuid,
 100             }
 101             stubs.api.collections().list().execute.return_value = {"items": []}
 102             stubs.api.containers().current().execute.return_value = {
 103                 "uuid": stubs.fake_container_uuid,
 104             }
 105             stubs.api.config()["StorageClasses"].items.return_value = {
 106                 "default": {
 107                     "Default": True
 108                 }
 109             }.items()
 110
 111             class CollectionExecute(object):
 112                 def __init__(self, exe):
 113                     self.exe = exe
 114                 def execute(self, num_retries=None):
 115                     return self.exe
 116
 117             def collection_createstub(created_collections, body, ensure_unique_name=None):
 118                 mt = body["manifest_text"].encode('utf-8')
 119                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 120                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 121                 created_collections[uuid] = {
 122                     "uuid": uuid,
 123                     "portable_data_hash": pdh,
 124                     "manifest_text": mt.decode('utf-8')
 125                 }
 126                 return CollectionExecute(created_collections[uuid])
 127
 128             def collection_getstub(created_collections, uuid):
 129                 for v in created_collections.values():
 130                     if uuid in (v["uuid"], v["portable_data_hash"]):
 131                         return CollectionExecute(v)
 132
 133             created_collections = {
 134                 "99999999999999999999999999999998+99": {
 135                     "uuid": "",
 136                     "portable_data_hash": "99999999999999999999999999999998+99",
 137                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 138                 },
 139                 "99999999999999999999999999999997+99": {
 140                     "uuid": "",
 141                     "portable_data_hash": "99999999999999999999999999999997+99",
 142                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 143                 },
 144                 "99999999999999999999999999999994+99": {
 145                     "uuid": "",
 146                     "portable_data_hash": "99999999999999999999999999999994+99",
 147                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 148                 },
 149                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 150                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 151                     "portable_data_hash": "999999999999999999999999999999d3+99",
 152                     "manifest_text": ""
 153                 },
 154                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 155                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 156                     "portable_data_hash": "999999999999999999999999999999d4+99",
 157                     "manifest_text": ""
 158                 },
 159                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 160                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 161                     "portable_data_hash": "999999999999999999999999999999d5+99",
 162                     "manifest_text": ""
 163                 },
 164                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 165                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 166                     "portable_data_hash": "999999999999999999999999999999d6+99",
 167                     "manifest_text": ""
 168                 }
 169             }
 170             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 171             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 172
 173             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 174             stubs.api.container_requests().create().execute.return_value = {
 175                 "uuid": stubs.expect_container_request_uuid,
 176                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 177                 "state": "Queued"
 178             }
 179
 180             cwd = os.getcwd()
 181             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 182             with open(filepath) as f:
 183                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 184                 expect_packed_workflow = yaml.load(f)
 185
 186             if wfpath is None:
 187                 wfpath = wfname
 188
 189             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 190             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 191             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 192
 193             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 194             expect_packed_workflow.update(stubs.git_info)
 195
 196             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 197
 198             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 199             if wfname == wfpath:
 200                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 201             else:
 202                 container_name = wfname
 203
 204             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 205             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 206
 207             stubs.expect_container_spec = {
 208                 'priority': 500,
 209                 'mounts': {
 210                     '/var/spool/cwl': {
 211                         'writable': True,
 212                         'kind': 'collection'
 213                     },
 214                     '/var/lib/cwl/workflow.json': {
 215                         'content': expect_packed_workflow,
 216                         'kind': 'json'
 217                     },
 218                     'stdout': {
 219                         'path': '/var/spool/cwl/cwl.output.json',
 220                         'kind': 'file'
 221                     },
 222                     '/var/lib/cwl/cwl.input.json': {
 223                         'kind': 'json',
 224                         'content': {
 225                             'y': {
 226                                 'basename': '99999999999999999999999999999998+99',
 227                                 'location': 'keep:99999999999999999999999999999998+99',
 228                                 'class': 'Directory'},
 229                             'x': {
 230                                 'basename': u'blorp.txt',
 231                                 'class': 'File',
 232                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 233                                 "size": 16
 234                             },
 235                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 236                                 {'basename': 'renamed.txt',
 237                                  'class': 'File',
 238                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 239                                  'size': 0
 240                                 }
 241                             ]}
 242                         },
 243                         'kind': 'json'
 244                     }
 245                 },
 246                 'secret_mounts': {},
 247                 'state': 'Committed',
 248                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 249                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 250                             '--eval-timeout=20', '--thread-count=0',
 251                             '--enable-reuse', "--collection-cache-size=256",
 252                             '--output-name=Output from workflow '+container_name,
 253                             '--debug', '--on-error=continue',
 254                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 255                 'name': container_name,
 256                 'container_image': '999999999999999999999999999999d3+99',
 257                 'output_name': 'Output from workflow %s' % (container_name),
 258                 'output_path': '/var/spool/cwl',
 259                 'cwd': '/var/spool/cwl',
 260                 'runtime_constraints': {
 261                     'API': True,
 262                     'vcpus': 1,
 263                     'ram': (1024+256)*1024*1024
 264                 },
 265                 'properties': stubs.git_props,
 266                 'use_existing': False,
 267                 'secret_mounts': {},
 268                 'environment': {},
 269             }
 270
 271             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 272             stubs.api.workflows().create().execute.return_value = {
 273                 "uuid": stubs.expect_workflow_uuid,
 274             }
 275             def update_mock(**kwargs):
 276                 stubs.updated_uuid = kwargs.get('uuid')
 277                 return mock.DEFAULT
 278             stubs.api.workflows().update.side_effect = update_mock
 279             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 280                 "uuid": stubs.updated_uuid,
 281             }
 282
 283             return func(self, stubs, *args, **kwargs)
 284         return wrapped
 285     return outer_wrapper
 286
 287 class TestSubmit(unittest.TestCase):
 288
 289     def setUp(self):
 290         cwltool.process._names = set()
 291         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 292
 293     def tearDown(self):
 294         root_logger = logging.getLogger('')
 295
 296         # Remove existing RuntimeStatusLoggingHandlers if they exist
 297         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 298         root_logger.handlers = handlers
 299
 300     @mock.patch("time.sleep")
 301     @stubs()
 302     def test_submit_invalid_runner_ram(self, stubs, tm):
 303         exited = arvados_cwl.main(
 304             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 305              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 306             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 307         self.assertEqual(exited, 1)
 308
 309
 310     @stubs()
 311     def test_submit_container(self, stubs):
 312         exited = arvados_cwl.main(
 313             ["--submit", "--no-wait", "--api=containers", "--debug",
 314                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 316
 317         stubs.api.collections().create.assert_has_calls([
 318             mock.call(body=JsonDiffMatcher({
 319                 'manifest_text':
 320                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 321                 'replication_desired': None,
 322                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 323             }), ensure_unique_name=False),
 324             mock.call(body=JsonDiffMatcher({
 325                 'manifest_text':
 326                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 327                 'replication_desired': None,
 328                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 329             }), ensure_unique_name=False),
 330             ])
 331
 332         expect_container = copy.deepcopy(stubs.expect_container_spec)
 333         stubs.api.container_requests().create.assert_called_with(
 334             body=JsonDiffMatcher(expect_container))
 335         self.assertEqual(stubs.capture_stdout.getvalue(),
 336                          stubs.expect_container_request_uuid + '\n')
 337         self.assertEqual(exited, 0)
 338
 339
 340     @stubs()
 341     def test_submit_container_tool(self, stubs):
 342         # test for issue #16139
 343         exited = arvados_cwl.main(
 344             ["--submit", "--no-wait", "--api=containers", "--debug",
 345                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 346             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 347
 348         self.assertEqual(stubs.capture_stdout.getvalue(),
 349                          stubs.expect_container_request_uuid + '\n')
 350         self.assertEqual(exited, 0)
 351
 352     @stubs()
 353     def test_submit_container_no_reuse(self, stubs):
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 356                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 358
 359         expect_container = copy.deepcopy(stubs.expect_container_spec)
 360         expect_container["command"] = [
 361             'arvados-cwl-runner', '--local', '--api=containers',
 362             '--no-log-timestamps', '--disable-validate', '--disable-color',
 363             '--eval-timeout=20', '--thread-count=0',
 364             '--disable-reuse', "--collection-cache-size=256",
 365             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 366             '--debug', '--on-error=continue',
 367             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 368         expect_container["use_existing"] = False
 369
 370         stubs.api.container_requests().create.assert_called_with(
 371             body=JsonDiffMatcher(expect_container))
 372         self.assertEqual(stubs.capture_stdout.getvalue(),
 373                          stubs.expect_container_request_uuid + '\n')
 374         self.assertEqual(exited, 0)
 375
 376     @stubs(('submit_wf_no_reuse.cwl', None))
 377     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 378         exited = arvados_cwl.main(
 379             ["--submit", "--no-wait", "--api=containers", "--debug",
 380              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 381             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 382         self.assertEqual(exited, 0)
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 386         expect_container["use_existing"] = False
 387         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 388             {
 389                 "class": "WorkReuse",
 390                 "enableReuse": False,
 391             },
 392                 {
 393                     "acrContainerImage": "999999999999999999999999999999d3+99",
 394                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 395                 }
 396         ]
 397         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 398
 399         stubs.api.container_requests().create.assert_called_with(
 400             body=JsonDiffMatcher(expect_container))
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403
 404
 405     @stubs()
 406     def test_submit_container_on_error(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 414                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 415                                        '--eval-timeout=20', '--thread-count=0',
 416                                        '--enable-reuse', "--collection-cache-size=256",
 417                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 418                                        '--debug', '--on-error=stop',
 419                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420
 421         stubs.api.container_requests().create.assert_called_with(
 422             body=JsonDiffMatcher(expect_container))
 423         self.assertEqual(stubs.capture_stdout.getvalue(),
 424                          stubs.expect_container_request_uuid + '\n')
 425         self.assertEqual(exited, 0)
 426
 427     @stubs()
 428     def test_submit_container_output_name(self, stubs):
 429         output_name = "test_output_name"
 430
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 433                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 438                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 439                                        '--eval-timeout=20', '--thread-count=0',
 440                                        '--enable-reuse', "--collection-cache-size=256",
 441                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 442                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["output_name"] = output_name
 444
 445         stubs.api.container_requests().create.assert_called_with(
 446             body=JsonDiffMatcher(expect_container))
 447         self.assertEqual(stubs.capture_stdout.getvalue(),
 448                          stubs.expect_container_request_uuid + '\n')
 449         self.assertEqual(exited, 0)
 450
 451     @stubs()
 452     def test_submit_storage_classes(self, stubs):
 453         exited = arvados_cwl.main(
 454             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 455                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 456             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 457
 458         expect_container = copy.deepcopy(stubs.expect_container_spec)
 459         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 460                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 461                                        '--eval-timeout=20', '--thread-count=0',
 462                                        '--enable-reuse', "--collection-cache-size=256",
 463                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 464                                        "--debug",
 465                                        "--storage-classes=foo", '--on-error=continue',
 466                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 467
 468         stubs.api.container_requests().create.assert_called_with(
 469             body=JsonDiffMatcher(expect_container))
 470         self.assertEqual(stubs.capture_stdout.getvalue(),
 471                          stubs.expect_container_request_uuid + '\n')
 472         self.assertEqual(exited, 0)
 473
 474     @stubs()
 475     def test_submit_multiple_storage_classes(self, stubs):
 476         exited = arvados_cwl.main(
 477             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 478                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 479             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 480
 481         expect_container = copy.deepcopy(stubs.expect_container_spec)
 482         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 483                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 484                                        '--eval-timeout=20', '--thread-count=0',
 485                                        '--enable-reuse', "--collection-cache-size=256",
 486                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 487                                        "--debug",
 488                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 489                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 490
 491         stubs.api.container_requests().create.assert_called_with(
 492             body=JsonDiffMatcher(expect_container))
 493         self.assertEqual(stubs.capture_stdout.getvalue(),
 494                          stubs.expect_container_request_uuid + '\n')
 495         self.assertEqual(exited, 0)
 496
 497     @stubs()
 498     def test_submit_invalid_empty_storage_classes(self, stubs):
 499         exited = arvados_cwl.main(
 500             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=",
 501                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 502             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 503
 504         expect_container = copy.deepcopy(stubs.expect_container_spec)
 505         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 506                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 507                                        '--eval-timeout=20', '--thread-count=0',
 508                                        '--enable-reuse', "--collection-cache-size=256",
 509                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 510                                        "--debug",
 511                                        '--on-error=continue',
 512                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 513
 514         stubs.api.container_requests().create.assert_called_with(
 515             body=JsonDiffMatcher(expect_container))
 516         self.assertEqual(stubs.capture_stdout.getvalue(),
 517                          stubs.expect_container_request_uuid + '\n')
 518         self.assertEqual(exited, 0)
 519
 520     @stubs()
 521     def test_submit_invalid_malformed_storage_classes(self, stubs):
 522         exited = arvados_cwl.main(
 523             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=,,,,,",
 524                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 525             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 526
 527         expect_container = copy.deepcopy(stubs.expect_container_spec)
 528         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 529                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 530                                        '--eval-timeout=20', '--thread-count=0',
 531                                        '--enable-reuse', "--collection-cache-size=256",
 532                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 533                                        "--debug",
 534                                        '--on-error=continue',
 535                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 536
 537         stubs.api.container_requests().create.assert_called_with(
 538             body=JsonDiffMatcher(expect_container))
 539         self.assertEqual(stubs.capture_stdout.getvalue(),
 540                          stubs.expect_container_request_uuid + '\n')
 541         self.assertEqual(exited, 0)
 542
 543     @stubs()
 544     def test_submit_duplicate_storage_classes(self, stubs):
 545         exited = arvados_cwl.main(
 546             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=,foo,bar,,foo,",
 547                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 548             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 549
 550         expect_container = copy.deepcopy(stubs.expect_container_spec)
 551         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 552                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 553                                        '--eval-timeout=20', '--thread-count=0',
 554                                        '--enable-reuse', "--collection-cache-size=256",
 555                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 556                                        "--debug",
 557                                        "--storage-classes=foo,bar", '--on-error=continue',
 558                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 559
 560         stubs.api.container_requests().create.assert_called_with(
 561             body=JsonDiffMatcher(expect_container))
 562         self.assertEqual(stubs.capture_stdout.getvalue(),
 563                          stubs.expect_container_request_uuid + '\n')
 564         self.assertEqual(exited, 0)
 565
 566     @mock.patch("cwltool.task_queue.TaskQueue")
 567     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 568     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 569     @stubs()
 570     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 571         final_output_c = arvados.collection.Collection()
 572         make_output.return_value = ({},final_output_c)
 573
 574         def set_final_output(job_order, output_callback, runtimeContext):
 575             output_callback({"out": "zzzzz"}, "success")
 576             return []
 577         job.side_effect = set_final_output
 578
 579         exited = arvados_cwl.main(
 580             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 581                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 582             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 583
 584         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 585         self.assertEqual(exited, 0)
 586
 587     @mock.patch("cwltool.task_queue.TaskQueue")
 588     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 589     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 590     @stubs()
 591     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 592         final_output_c = arvados.collection.Collection()
 593         make_output.return_value = ({},final_output_c)
 594         stubs.api.config().get.return_value = {"default": {"Default": True}}
 595
 596         def set_final_output(job_order, output_callback, runtimeContext):
 597             output_callback({"out": "zzzzz"}, "success")
 598             return []
 599         job.side_effect = set_final_output
 600
 601         exited = arvados_cwl.main(
 602             ["--debug", "--local", "--disable-git",
 603                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 604             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 605
 606         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 607         self.assertEqual(exited, 0)
 608
 609     @mock.patch("cwltool.task_queue.TaskQueue")
 610     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 611     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 612     @stubs()
 613     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 614         final_output_c = arvados.collection.Collection()
 615         make_output.return_value = ({},final_output_c)
 616
 617         def set_final_output(job_order, output_callback, runtimeContext):
 618             output_callback({"out": "zzzzz"}, "success")
 619             return []
 620         job.side_effect = set_final_output
 621
 622         exited = arvados_cwl.main(
 623             ["--debug", "--local", "--disable-git",
 624                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 625             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 626
 627         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 628         self.assertEqual(exited, 0)
 629
 630     @stubs()
 631     def test_submit_container_output_ttl(self, stubs):
 632         exited = arvados_cwl.main(
 633             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 634                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 635             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 636
 637         expect_container = copy.deepcopy(stubs.expect_container_spec)
 638         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 639                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 640                                        '--eval-timeout=20', '--thread-count=0',
 641                                        '--enable-reuse', "--collection-cache-size=256",
 642                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 643                                        '--debug',
 644                                        '--on-error=continue',
 645                                        "--intermediate-output-ttl=3600",
 646                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 647
 648         stubs.api.container_requests().create.assert_called_with(
 649             body=JsonDiffMatcher(expect_container))
 650         self.assertEqual(stubs.capture_stdout.getvalue(),
 651                          stubs.expect_container_request_uuid + '\n')
 652         self.assertEqual(exited, 0)
 653
 654     @stubs()
 655     def test_submit_container_trash_intermediate(self, stubs):
 656         exited = arvados_cwl.main(
 657             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 658                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 659             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 660
 661
 662         expect_container = copy.deepcopy(stubs.expect_container_spec)
 663         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 664                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 665                                        '--eval-timeout=20', '--thread-count=0',
 666                                        '--enable-reuse', "--collection-cache-size=256",
 667                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 668                                        '--debug', '--on-error=continue',
 669                                        "--trash-intermediate",
 670                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 671
 672         stubs.api.container_requests().create.assert_called_with(
 673             body=JsonDiffMatcher(expect_container))
 674         self.assertEqual(stubs.capture_stdout.getvalue(),
 675                          stubs.expect_container_request_uuid + '\n')
 676         self.assertEqual(exited, 0)
 677
 678     @stubs()
 679     def test_submit_container_output_tags(self, stubs):
 680         output_tags = "tag0,tag1,tag2"
 681
 682         exited = arvados_cwl.main(
 683             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 684                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 685             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 686
 687         expect_container = copy.deepcopy(stubs.expect_container_spec)
 688         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 689                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 690                                        '--eval-timeout=20', '--thread-count=0',
 691                                        '--enable-reuse', "--collection-cache-size=256",
 692                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 693                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 694                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 695
 696         stubs.api.container_requests().create.assert_called_with(
 697             body=JsonDiffMatcher(expect_container))
 698         self.assertEqual(stubs.capture_stdout.getvalue(),
 699                          stubs.expect_container_request_uuid + '\n')
 700         self.assertEqual(exited, 0)
 701
 702     @stubs()
 703     def test_submit_container_runner_ram(self, stubs):
 704         exited = arvados_cwl.main(
 705             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 706                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 707             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 708
 709         expect_container = copy.deepcopy(stubs.expect_container_spec)
 710         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 711         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 712                 {
 713                     "acrContainerImage": "999999999999999999999999999999d3+99",
 714                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 715                     "ramMin": 2048
 716                 }
 717         ]
 718
 719         stubs.api.container_requests().create.assert_called_with(
 720             body=JsonDiffMatcher(expect_container))
 721         self.assertEqual(stubs.capture_stdout.getvalue(),
 722                          stubs.expect_container_request_uuid + '\n')
 723         self.assertEqual(exited, 0)
 724
 725     @mock.patch("arvados.collection.CollectionReader")
 726     @mock.patch("time.sleep")
 727     @stubs()
 728     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 729         collectionReader().exists.return_value = True
 730         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 731         exited = arvados_cwl.main(
 732             ["--submit", "--no-wait", "--api=containers", "--debug",
 733              "tests/wf/submit_keepref_wf.cwl"],
 734             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 735         self.assertEqual(exited, 0)
 736
 737     @mock.patch("arvados.collection.CollectionReader")
 738     @mock.patch("time.sleep")
 739     @stubs()
 740     def test_submit_keepref(self, stubs, tm, reader):
 741         with open("tests/wf/expect_arvworkflow.cwl") as f:
 742             reader().open().__enter__().read.return_value = f.read()
 743
 744         exited = arvados_cwl.main(
 745             ["--submit", "--no-wait", "--api=containers", "--debug",
 746              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 747             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 748
 749         expect_container = {
 750             'priority': 500,
 751             'mounts': {
 752                 '/var/spool/cwl': {
 753                     'writable': True,
 754                     'kind': 'collection'
 755                 },
 756                 'stdout': {
 757                     'path': '/var/spool/cwl/cwl.output.json',
 758                     'kind': 'file'
 759                 },
 760                 '/var/lib/cwl/workflow': {
 761                     'portable_data_hash': '99999999999999999999999999999994+99',
 762                     'kind': 'collection'
 763                 },
 764                 '/var/lib/cwl/cwl.input.json': {
 765                     'content': {
 766                         'x': 'XxX'
 767                     },
 768                     'kind': 'json'
 769                 }
 770             }, 'state': 'Committed',
 771             'output_path': '/var/spool/cwl',
 772             'name': 'expect_arvworkflow.cwl#main',
 773             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 774             'container_image': '999999999999999999999999999999d3+99',
 775             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 776                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 777                         '--eval-timeout=20', '--thread-count=0',
 778                         '--enable-reuse', "--collection-cache-size=256",
 779                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 780                         '--debug', '--on-error=continue',
 781                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 782             'cwd': '/var/spool/cwl',
 783             'runtime_constraints': {
 784                 'API': True,
 785                 'vcpus': 1,
 786                 'ram': 1342177280
 787             },
 788             'use_existing': False,
 789             'properties': {},
 790             'secret_mounts': {},
 791             'environment': {},
 792         }
 793
 794         stubs.api.container_requests().create.assert_called_with(
 795             body=JsonDiffMatcher(expect_container))
 796         self.assertEqual(stubs.capture_stdout.getvalue(),
 797                          stubs.expect_container_request_uuid + '\n')
 798         self.assertEqual(exited, 0)
 799
 800     @mock.patch("time.sleep")
 801     @stubs()
 802     def test_submit_arvworkflow(self, stubs, tm):
 803         with open("tests/wf/expect_arvworkflow.cwl") as f:
 804             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 805
 806         exited = arvados_cwl.main(
 807             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 808              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 809             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 810
 811         expect_container = {
 812             'priority': 500,
 813             'mounts': {
 814                 '/var/spool/cwl': {
 815                     'writable': True,
 816                     'kind': 'collection'
 817                 },
 818                 'stdout': {
 819                     'path': '/var/spool/cwl/cwl.output.json',
 820                     'kind': 'file'
 821                 },
 822                 '/var/lib/cwl/workflow.json': {
 823                     'kind': 'json',
 824                     'content': {
 825                         'cwlVersion': 'v1.0',
 826                         'label': 'a test workflow',
 827                         '$graph': [
 828                             {
 829                                 'id': '#main',
 830                                 'inputs': [
 831                                     {'type': 'string', 'id': '#main/x'}
 832                                 ],
 833                                 'steps': [
 834                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 835                                      'run': '#submit_tool.cwl',
 836                                      'id': '#main/step1',
 837                                      'out': []}
 838                                 ],
 839                                 'class': 'Workflow',
 840                                 'outputs': []
 841                             },
 842                             {
 843                                 'inputs': [
 844                                     {
 845                                         'inputBinding': {'position': 1},
 846                                         'type': 'string',
 847                                         'id': '#submit_tool.cwl/x'}
 848                                 ],
 849                                 'requirements': [
 850                                     {
 851                                         'dockerPull': 'debian:buster-slim',
 852                                         'class': 'DockerRequirement'
 853                                     }
 854                                 ],
 855                                 'id': '#submit_tool.cwl',
 856                                 'outputs': [],
 857                                 'baseCommand': 'cat',
 858                                 'class': 'CommandLineTool'
 859                             }
 860                         ]
 861                     }
 862                 },
 863                 '/var/lib/cwl/cwl.input.json': {
 864                     'content': {
 865                         'x': 'XxX'
 866                     },
 867                     'kind': 'json'
 868                 }
 869             }, 'state': 'Committed',
 870             'output_path': '/var/spool/cwl',
 871             'name': 'a test workflow',
 872             'container_image': "999999999999999999999999999999d3+99",
 873             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 874                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 875                         '--eval-timeout=20', '--thread-count=0',
 876                         '--enable-reuse', "--collection-cache-size=256",
 877                         "--output-name=Output from workflow a test workflow",
 878                         '--debug', '--on-error=continue',
 879                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 880             'output_name': 'Output from workflow a test workflow',
 881             'cwd': '/var/spool/cwl',
 882             'runtime_constraints': {
 883                 'API': True,
 884                 'vcpus': 1,
 885                 'ram': 1342177280
 886             },
 887             'use_existing': False,
 888             'properties': {
 889                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 890             },
 891             'secret_mounts': {},
 892             'environment': {},
 893         }
 894
 895         stubs.api.container_requests().create.assert_called_with(
 896             body=JsonDiffMatcher(expect_container))
 897         self.assertEqual(stubs.capture_stdout.getvalue(),
 898                          stubs.expect_container_request_uuid + '\n')
 899         self.assertEqual(exited, 0)
 900
 901     @stubs(('hello container 123', 'submit_wf.cwl'))
 902     def test_submit_container_name(self, stubs):
 903         exited = arvados_cwl.main(
 904             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 905                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 906             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 907
 908         expect_container = copy.deepcopy(stubs.expect_container_spec)
 909
 910         stubs.api.container_requests().create.assert_called_with(
 911             body=JsonDiffMatcher(expect_container))
 912         self.assertEqual(stubs.capture_stdout.getvalue(),
 913                          stubs.expect_container_request_uuid + '\n')
 914         self.assertEqual(exited, 0)
 915
 916     @stubs()
 917     def test_submit_missing_input(self, stubs):
 918         exited = arvados_cwl.main(
 919             ["--submit", "--no-wait", "--api=containers", "--debug",
 920              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 921             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 922         self.assertEqual(exited, 0)
 923
 924         exited = arvados_cwl.main(
 925             ["--submit", "--no-wait", "--api=containers", "--debug",
 926              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 927             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 928         self.assertEqual(exited, 1)
 929
 930     @stubs()
 931     def test_submit_container_project(self, stubs):
 932         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 933         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 934         exited = arvados_cwl.main(
 935             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 936                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 937             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 938
 939         expect_container = copy.deepcopy(stubs.expect_container_spec)
 940         expect_container["owner_uuid"] = project_uuid
 941         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 942                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 943                                        "--eval-timeout=20", "--thread-count=0",
 944                                        '--enable-reuse', "--collection-cache-size=256",
 945                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 946                                        '--debug',
 947                                        '--on-error=continue',
 948                                        '--project-uuid='+project_uuid,
 949                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 950
 951         stubs.api.container_requests().create.assert_called_with(
 952             body=JsonDiffMatcher(expect_container))
 953         self.assertEqual(stubs.capture_stdout.getvalue(),
 954                          stubs.expect_container_request_uuid + '\n')
 955         self.assertEqual(exited, 0)
 956
 957     @stubs()
 958     def test_submit_container_eval_timeout(self, stubs):
 959         exited = arvados_cwl.main(
 960             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 961                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 962             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 963
 964         expect_container = copy.deepcopy(stubs.expect_container_spec)
 965         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 966                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 967                                        '--eval-timeout=60.0', '--thread-count=0',
 968                                        '--enable-reuse', "--collection-cache-size=256",
 969                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 970                                        '--debug', '--on-error=continue',
 971                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 972
 973         stubs.api.container_requests().create.assert_called_with(
 974             body=JsonDiffMatcher(expect_container))
 975         self.assertEqual(stubs.capture_stdout.getvalue(),
 976                          stubs.expect_container_request_uuid + '\n')
 977         self.assertEqual(exited, 0)
 978
 979     @stubs()
 980     def test_submit_container_collection_cache(self, stubs):
 981         exited = arvados_cwl.main(
 982             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 983                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 984             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 985
 986         expect_container = copy.deepcopy(stubs.expect_container_spec)
 987         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 988                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 989                                        '--eval-timeout=20', '--thread-count=0',
 990                                        '--enable-reuse', "--collection-cache-size=500",
 991                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 992                                        '--debug', '--on-error=continue',
 993                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 994         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 995
 996         stubs.api.container_requests().create.assert_called_with(
 997             body=JsonDiffMatcher(expect_container))
 998         self.assertEqual(stubs.capture_stdout.getvalue(),
 999                          stubs.expect_container_request_uuid + '\n')
1000         self.assertEqual(exited, 0)
1001
1002     @stubs()
1003     def test_submit_container_thread_count(self, stubs):
1004         exited = arvados_cwl.main(
1005             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1006                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1007             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1008
1009         expect_container = copy.deepcopy(stubs.expect_container_spec)
1010         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1011                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1012                                        '--eval-timeout=20', '--thread-count=20',
1013                                        '--enable-reuse', "--collection-cache-size=256",
1014                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1015                                        '--debug', '--on-error=continue',
1016                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1017
1018         stubs.api.container_requests().create.assert_called_with(
1019             body=JsonDiffMatcher(expect_container))
1020         self.assertEqual(stubs.capture_stdout.getvalue(),
1021                          stubs.expect_container_request_uuid + '\n')
1022         self.assertEqual(exited, 0)
1023
1024     @stubs()
1025     def test_submit_container_runner_image(self, stubs):
1026         exited = arvados_cwl.main(
1027             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1028                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1029             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1030
1031         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1032         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1033                 {
1034                     "acrContainerImage": "999999999999999999999999999999d5+99",
1035                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1036                 }
1037         ]
1038
1039         expect_container = copy.deepcopy(stubs.expect_container_spec)
1040         stubs.api.container_requests().create.assert_called_with(
1041             body=JsonDiffMatcher(expect_container))
1042         self.assertEqual(stubs.capture_stdout.getvalue(),
1043                          stubs.expect_container_request_uuid + '\n')
1044         self.assertEqual(exited, 0)
1045
1046     @stubs()
1047     def test_submit_priority(self, stubs):
1048         exited = arvados_cwl.main(
1049             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1050                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1051             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1052
1053         stubs.expect_container_spec["priority"] = 669
1054
1055         expect_container = copy.deepcopy(stubs.expect_container_spec)
1056         stubs.api.container_requests().create.assert_called_with(
1057             body=JsonDiffMatcher(expect_container))
1058         self.assertEqual(stubs.capture_stdout.getvalue(),
1059                          stubs.expect_container_request_uuid + '\n')
1060         self.assertEqual(exited, 0)
1061
1062     @stubs(('submit_wf_runner_resources.cwl', None))
1063     def test_submit_wf_runner_resources(self, stubs):
1064         exited = arvados_cwl.main(
1065             ["--submit", "--no-wait", "--api=containers", "--debug",
1066                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1067             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1068
1069         expect_container = copy.deepcopy(stubs.expect_container_spec)
1070         expect_container["runtime_constraints"] = {
1071             "API": True,
1072             "vcpus": 2,
1073             "ram": (2000+512) * 2**20
1074         }
1075         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1076             {
1077                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1078                 "acrContainerImage": "999999999999999999999999999999d3+99",
1079                 "coresMin": 2,
1080                 "ramMin": 2000,
1081                 "keep_cache": 512
1082             }
1083         ]
1084         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1085         #    "arv": "http://arvados.org/cwl#",
1086         #}
1087         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1088         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1089
1090         stubs.api.container_requests().create.assert_called_with(
1091             body=JsonDiffMatcher(expect_container))
1092         self.assertEqual(stubs.capture_stdout.getvalue(),
1093                          stubs.expect_container_request_uuid + '\n')
1094         self.assertEqual(exited, 0)
1095
1096     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1097     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1098     @mock.patch("arvados.api")
1099     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1100         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1101
1102         arvrunner = mock.MagicMock()
1103         arvrunner.project_uuid = ""
1104         api.return_value = mock.MagicMock()
1105         arvrunner.api = api.return_value
1106         arvrunner.runtimeContext.match_local_docker = False
1107         arvrunner.api.links().list().execute.side_effect = itertools.cycle([
1108             {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
1109                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1110                         "link_class": "docker_image_repo+tag",
1111                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1112                         "owner_uuid": "",
1113                         "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
1114                         "properties": {"image_timestamp": ""}}]},
1115             {"items": []},
1116             {"items": []},
1117             {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
1118                         "head_uuid": "",
1119                         "link_class": "docker_image_hash",
1120                         "name": "123456",
1121                         "owner_uuid": "",
1122                         "uuid": "zzzzz-o0j2j-arvadosjobshash",
1123                         "properties": {"image_timestamp": ""}}]},
1124             {"items": []},
1125             {"items": []},
1126         ])
1127         find_one_image_hash.return_value = "123456"
1128
1129         arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
1130             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1131                         "owner_uuid": "",
1132                         "manifest_text": "",
1133                         "created_at": "2023-08-25T12:34:55.012345Z",
1134                         "properties": {}}]},
1135             {"items": []},
1136             {"items": []},
1137         ])
1138         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1139         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1140                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1141
1142         self.assertEqual("9999999999999999999999999999999b+99",
1143                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1144
1145
1146     @stubs()
1147     def test_submit_secrets(self, stubs):
1148         exited = arvados_cwl.main(
1149             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1150                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1151             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1152
1153         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1154         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1155
1156         expect_container = {
1157             "command": [
1158                 "arvados-cwl-runner",
1159                 "--local",
1160                 "--api=containers",
1161                 "--no-log-timestamps",
1162                 "--disable-validate",
1163                 "--disable-color",
1164                 "--eval-timeout=20",
1165                 '--thread-count=0',
1166                 "--enable-reuse",
1167                 "--collection-cache-size=256",
1168                 "--output-name=Output from workflow secret_wf.cwl",
1169                 "--debug",
1170                 "--on-error=continue",
1171                 "/var/lib/cwl/workflow.json#main",
1172                 "/var/lib/cwl/cwl.input.json"
1173             ],
1174             "container_image": "999999999999999999999999999999d3+99",
1175             "cwd": "/var/spool/cwl",
1176             "mounts": {
1177                 "/var/lib/cwl/cwl.input.json": {
1178                     "content": {
1179                         "pw": {
1180                             "$include": "/secrets/s0"
1181                         }
1182                     },
1183                     "kind": "json"
1184                 },
1185                 "/var/lib/cwl/workflow.json": {
1186                     "content": {
1187                         "$graph": [
1188                             {
1189                                 "class": "Workflow",
1190                                 "hints": [
1191                                     {
1192                                     "class": "DockerRequirement",
1193                                     "dockerPull": "debian:buster-slim",
1194                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1195                                     },
1196                                     {
1197                                         "class": "http://commonwl.org/cwltool#Secrets",
1198                                         "secrets": [
1199                                             "#main/pw"
1200                                         ]
1201                                     },
1202                                     {
1203                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1204                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1205                                     }
1206                                 ],
1207                                 "id": "#main",
1208                                 "inputs": [
1209                                     {
1210                                         "id": "#main/pw",
1211                                         "type": "string"
1212                                     }
1213                                 ],
1214                                 "outputs": [
1215                                     {
1216                                         "id": "#main/out",
1217                                         "outputSource": "#main/step/out",
1218                                         "type": "File"
1219                                     }
1220                                 ],
1221                                 "requirements": [
1222                                     {
1223                                         "class": "SubworkflowFeatureRequirement"
1224                                     }
1225                                 ],
1226                                 "steps": [
1227                                     {
1228                                         "id": "#main/secret_wf.cwl",
1229                                         "in": [
1230                                             {
1231                                                 "id": "#main/step/pw",
1232                                                 "source": "#main/pw"
1233                                             }
1234                                         ],
1235                                         "label": "secret_wf.cwl",
1236                                         "out": [
1237                                             {"id": "#main/step/out"}
1238                                         ],
1239                                         "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
1240                                     }
1241                                 ]
1242                             }
1243                         ],
1244                         "cwlVersion": "v1.2"
1245                     },
1246                     "kind": "json"
1247                 },
1248                 "/var/spool/cwl": {
1249                     "kind": "collection",
1250                     "writable": True
1251                 },
1252                 "stdout": {
1253                     "kind": "file",
1254                     "path": "/var/spool/cwl/cwl.output.json"
1255                 }
1256             },
1257             "name": "secret_wf.cwl",
1258             "output_name": "Output from workflow secret_wf.cwl",
1259             "output_path": "/var/spool/cwl",
1260             "priority": 500,
1261             "properties": {},
1262             "runtime_constraints": {
1263                 "API": True,
1264                 "ram": 1342177280,
1265                 "vcpus": 1
1266             },
1267             "secret_mounts": {
1268                 "/secrets/s0": {
1269                     "content": "blorp",
1270                     "kind": "text"
1271                 }
1272             },
1273             "state": "Committed",
1274             "use_existing": False,
1275             "environment": {}
1276         }
1277
1278         stubs.api.container_requests().create.assert_called_with(
1279             body=JsonDiffMatcher(expect_container))
1280         self.assertEqual(stubs.capture_stdout.getvalue(),
1281                          stubs.expect_container_request_uuid + '\n')
1282         self.assertEqual(exited, 0)
1283
1284     @stubs()
1285     def test_submit_request_uuid(self, stubs):
1286         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1287         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1288
1289         stubs.api.container_requests().update().execute.return_value = {
1290             "uuid": stubs.expect_container_request_uuid,
1291             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1292             "state": "Queued"
1293         }
1294
1295         exited = arvados_cwl.main(
1296             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1297                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1298             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1299
1300         stubs.api.container_requests().update.assert_called_with(
1301             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1302         self.assertEqual(stubs.capture_stdout.getvalue(),
1303                          stubs.expect_container_request_uuid + '\n')
1304         self.assertEqual(exited, 0)
1305
1306     @stubs()
1307     def test_submit_container_cluster_id(self, stubs):
1308         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1309
1310         exited = arvados_cwl.main(
1311             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1312                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1313             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1314
1315         expect_container = copy.deepcopy(stubs.expect_container_spec)
1316
1317         stubs.api.container_requests().create.assert_called_with(
1318             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1319         self.assertEqual(stubs.capture_stdout.getvalue(),
1320                          stubs.expect_container_request_uuid + '\n')
1321         self.assertEqual(exited, 0)
1322
1323     @stubs()
1324     def test_submit_validate_cluster_id(self, stubs):
1325         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1326         exited = arvados_cwl.main(
1327             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1328              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1329             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1330         self.assertEqual(exited, 1)
1331
1332     @stubs()
1333     def test_submit_validate_project_uuid(self, stubs):
1334         # Fails with bad cluster prefix
1335         exited = arvados_cwl.main(
1336             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1337              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1338             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1339         self.assertEqual(exited, 1)
1340
1341         # Project lookup fails
1342         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1343         exited = arvados_cwl.main(
1344             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1345              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1346             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1347         self.assertEqual(exited, 1)
1348
1349         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1350         exited = arvados_cwl.main(
1351             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1352              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1353             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1354         self.assertEqual(exited, 0)
1355
1356
1357     @mock.patch("arvados.collection.CollectionReader")
1358     @stubs()
1359     def test_submit_uuid_inputs(self, stubs, collectionReader):
1360         collectionReader().exists.return_value = True
1361         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1362         def list_side_effect(**kwargs):
1363             m = mock.MagicMock()
1364             if "count" in kwargs:
1365                 m.execute.return_value = {"items": [
1366                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1367                 ]}
1368             else:
1369                 m.execute.return_value = {"items": []}
1370             return m
1371         stubs.api.collections().list.side_effect = list_side_effect
1372         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1373
1374         exited = arvados_cwl.main(
1375             ["--submit", "--no-wait", "--api=containers", "--debug",
1376                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1377             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1378
1379         expect_container = copy.deepcopy(stubs.expect_container_spec)
1380         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1381         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1382         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1383
1384         stubs.api.collections().list.assert_has_calls([
1385             mock.call(count='none',
1386                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1387                       select=['uuid', 'portable_data_hash'])])
1388         stubs.api.container_requests().create.assert_called_with(
1389             body=JsonDiffMatcher(expect_container))
1390         self.assertEqual(stubs.capture_stdout.getvalue(),
1391                          stubs.expect_container_request_uuid + '\n')
1392         self.assertEqual(exited, 0)
1393
1394     @stubs()
1395     def test_submit_mismatched_uuid_inputs(self, stubs):
1396         def list_side_effect(**kwargs):
1397             m = mock.MagicMock()
1398             if "count" in kwargs:
1399                 m.execute.return_value = {"items": [
1400                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1401                 ]}
1402             else:
1403                 m.execute.return_value = {"items": []}
1404             return m
1405         stubs.api.collections().list.side_effect = list_side_effect
1406
1407         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1408             capture_stderr = StringIO()
1409             cwltool_logger = logging.getLogger('cwltool')
1410             stderr_logger = logging.StreamHandler(capture_stderr)
1411             cwltool_logger.addHandler(stderr_logger)
1412
1413             try:
1414                 exited = arvados_cwl.main(
1415                     ["--submit", "--no-wait", "--api=containers", "--debug",
1416                         "tests/wf/submit_wf.cwl", infile],
1417                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1418
1419                 self.assertEqual(exited, 1)
1420                 self.assertRegex(
1421                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1422                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1423             finally:
1424                 cwltool_logger.removeHandler(stderr_logger)
1425
1426     @mock.patch("arvados.collection.CollectionReader")
1427     @stubs()
1428     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1429         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1430         capture_stderr = StringIO()
1431
1432         cwltool_logger = logging.getLogger('cwltool')
1433         stderr_logger = logging.StreamHandler(capture_stderr)
1434         cwltool_logger.addHandler(stderr_logger)
1435
1436         exited = arvados_cwl.main(
1437             ["--submit", "--no-wait", "--api=containers", "--debug",
1438                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1439             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1440
1441         try:
1442             self.assertEqual(exited, 1)
1443             self.assertRegex(
1444                 capture_stderr.getvalue(),
1445                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1446         finally:
1447             cwltool_logger.removeHandler(stderr_logger)
1448
1449     @stubs(('submit_wf_process_properties.cwl', None))
1450     def test_submit_set_process_properties(self, stubs):
1451         exited = arvados_cwl.main(
1452             ["--submit", "--no-wait", "--api=containers", "--debug",
1453                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1454             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1455
1456         expect_container = copy.deepcopy(stubs.expect_container_spec)
1457
1458         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1459             {
1460                 "class": "http://arvados.org/cwl#ProcessProperties",
1461                 "processProperties": [
1462                     {"propertyName": "baz",
1463                      "propertyValue": "$(inputs.x.basename)"},
1464                     {"propertyName": "foo",
1465                      "propertyValue": "bar"},
1466                     {"propertyName": "quux",
1467                      "propertyValue": {
1468                          "q1": 1,
1469                          "q2": 2
1470                      }
1471                     }
1472                 ],
1473             },
1474             {
1475                 "acrContainerImage": "999999999999999999999999999999d3+99",
1476                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1477             }
1478         ]
1479         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1480         #    "arv": "http://arvados.org/cwl#"
1481         #}
1482
1483         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1484
1485         expect_container["properties"].update({
1486             "baz": "blorp.txt",
1487             "foo": "bar",
1488             "quux": {
1489                 "q1": 1,
1490                 "q2": 2
1491             }
1492         })
1493
1494         stubs.api.container_requests().create.assert_called_with(
1495             body=JsonDiffMatcher(expect_container))
1496         self.assertEqual(stubs.capture_stdout.getvalue(),
1497                          stubs.expect_container_request_uuid + '\n')
1498         self.assertEqual(exited, 0)
1499
1500
1501     @stubs()
1502     def test_submit_enable_preemptible(self, stubs):
1503         exited = arvados_cwl.main(
1504             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1505                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1506             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1507
1508         expect_container = copy.deepcopy(stubs.expect_container_spec)
1509         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1510                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1511                                        '--eval-timeout=20', '--thread-count=0',
1512                                        '--enable-reuse', "--collection-cache-size=256",
1513                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1514                                        '--debug', '--on-error=continue',
1515                                        '--enable-preemptible',
1516                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1517
1518         stubs.api.container_requests().create.assert_called_with(
1519             body=JsonDiffMatcher(expect_container))
1520         self.assertEqual(stubs.capture_stdout.getvalue(),
1521                          stubs.expect_container_request_uuid + '\n')
1522         self.assertEqual(exited, 0)
1523
1524     @stubs()
1525     def test_submit_disable_preemptible(self, stubs):
1526         exited = arvados_cwl.main(
1527             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1528                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1529             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1530
1531         expect_container = copy.deepcopy(stubs.expect_container_spec)
1532         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1533                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1534                                        '--eval-timeout=20', '--thread-count=0',
1535                                        '--enable-reuse', "--collection-cache-size=256",
1536                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1537                                        '--debug', '--on-error=continue',
1538                                        '--disable-preemptible',
1539                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1540
1541         stubs.api.container_requests().create.assert_called_with(
1542             body=JsonDiffMatcher(expect_container))
1543         self.assertEqual(stubs.capture_stdout.getvalue(),
1544                          stubs.expect_container_request_uuid + '\n')
1545         self.assertEqual(exited, 0)
1546
1547     @stubs()
1548     def test_submit_container_prefer_cached_downloads(self, stubs):
1549         exited = arvados_cwl.main(
1550             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1551                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1552             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1553
1554         expect_container = copy.deepcopy(stubs.expect_container_spec)
1555         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1556                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1557                                        '--eval-timeout=20', '--thread-count=0',
1558                                        '--enable-reuse', "--collection-cache-size=256",
1559                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1560                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1561                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1562
1563         stubs.api.container_requests().create.assert_called_with(
1564             body=JsonDiffMatcher(expect_container))
1565         self.assertEqual(stubs.capture_stdout.getvalue(),
1566                          stubs.expect_container_request_uuid + '\n')
1567         self.assertEqual(exited, 0)
1568
1569     @stubs()
1570     def test_submit_container_varying_url_params(self, stubs):
1571         exited = arvados_cwl.main(
1572             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1573                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1574             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1575
1576         expect_container = copy.deepcopy(stubs.expect_container_spec)
1577         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1578                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1579                                        '--eval-timeout=20', '--thread-count=0',
1580                                        '--enable-reuse', "--collection-cache-size=256",
1581                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1582                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1583                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1584
1585         stubs.api.container_requests().create.assert_called_with(
1586             body=JsonDiffMatcher(expect_container))
1587         self.assertEqual(stubs.capture_stdout.getvalue(),
1588                          stubs.expect_container_request_uuid + '\n')
1589         self.assertEqual(exited, 0)
1590
1591     @mock.patch("boto3.session.Session")
1592     @stubs()
1593     def test_submit_defer_s3_download(self, stubs, botosession):
1594
1595         sessionmock = mock.MagicMock(region_name='us-east-2')
1596         botosession.return_value = sessionmock
1597
1598         CredsTuple = collections.namedtuple('CredsTuple', ['access_key', 'secret_key'])
1599
1600         sessionmock.get_credentials.return_value = CredsTuple('123key', '789secret')
1601
1602         exited = arvados_cwl.main(
1603             ["--submit", "--no-wait", "--api=containers", "--debug", "--defer-download",
1604                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_s3.json"],
1605             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1606
1607         expect_container = copy.deepcopy(stubs.expect_container_spec)
1608
1609         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['x']['location'] = 's3://examplebucket/blorp.txt'
1610         del expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['x']['size']
1611         expect_container['environment']['AWS_SHARED_CREDENTIALS_FILE'] = '/var/lib/cwl/.aws/credentials'
1612         expect_container['environment']['AWS_CONFIG_FILE'] = '/var/lib/cwl/.aws/config'
1613         expect_container['secret_mounts'] = {
1614             "/var/lib/cwl/.aws/credentials": {
1615                 "content": "[default]\naws_access_key_id = 123key\naws_secret_access_key = 789secret\n",
1616                 "kind": "text"
1617             },
1618             "/var/lib/cwl/.aws/config": {
1619                 "content": "[default]\nregion = us-east-2\n",
1620                 "kind": "text"
1621             }
1622         }
1623         stubs.api.container_requests().create.assert_called_with(
1624             body=JsonDiffMatcher(expect_container))
1625         self.assertEqual(stubs.capture_stdout.getvalue(),
1626                          stubs.expect_container_request_uuid + '\n')
1627         self.assertEqual(exited, 0)
1628
1629     @mock.patch("boto3.session.Session")
1630     @stubs()
1631     def test_submit_defer_s3_download_no_credential_capture(self, stubs, botosession):
1632
1633         sessionmock = mock.MagicMock(region_name='us-east-2')
1634         botosession.return_value = sessionmock
1635
1636         exited = arvados_cwl.main(
1637             ["--submit", "--no-wait", "--api=containers", "--debug", "--defer-download", "--disable-aws-credential-capture",
1638                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_s3.json"],
1639             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1640
1641         expect_container = copy.deepcopy(stubs.expect_container_spec)
1642
1643         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['x']['location'] = 's3://examplebucket/blorp.txt'
1644         del expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['x']['size']
1645         stubs.api.container_requests().create.assert_called_with(
1646             body=JsonDiffMatcher(expect_container))
1647         self.assertEqual(stubs.capture_stdout.getvalue(),
1648                          stubs.expect_container_request_uuid + '\n')
1649         self.assertEqual(exited, 0)
1650
1651         sessionmock.get_credentials.assert_not_called()
1652
1653 class TestCreateWorkflow(unittest.TestCase):
1654     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1655     expect_workflow = StripYAMLComments(
1656         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1657     expect_workflow_altname = StripYAMLComments(
1658         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1659
1660     def setUp(self):
1661         cwltool.process._names = set()
1662         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1663
1664     def tearDown(self):
1665         root_logger = logging.getLogger('')
1666
1667         # Remove existing RuntimeStatusLoggingHandlers if they exist
1668         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1669         root_logger.handlers = handlers
1670
1671     @stubs()
1672     def test_create(self, stubs):
1673         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1674         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1675
1676         exited = arvados_cwl.main(
1677             ["--create-workflow", "--debug",
1678              "--api=containers",
1679              "--project-uuid", project_uuid,
1680              "--disable-git",
1681              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1682             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1683
1684         stubs.api.pipeline_templates().create.refute_called()
1685         stubs.api.container_requests().create.refute_called()
1686
1687         body = {
1688             "workflow": {
1689                 "owner_uuid": project_uuid,
1690                 "name": "submit_wf.cwl",
1691                 "description": "",
1692                 "definition": self.expect_workflow,
1693             }
1694         }
1695         stubs.api.workflows().create.assert_called_with(
1696             body=JsonDiffMatcher(body))
1697
1698         self.assertEqual(stubs.capture_stdout.getvalue(),
1699                          stubs.expect_workflow_uuid + '\n')
1700         self.assertEqual(exited, 0)
1701
1702     @stubs()
1703     def test_create_name(self, stubs):
1704         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1705         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1706
1707         exited = arvados_cwl.main(
1708             ["--create-workflow", "--debug",
1709              "--api=containers",
1710              "--project-uuid", project_uuid,
1711              "--name", "testing 123",
1712              "--disable-git",
1713              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1714             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1715
1716         stubs.api.pipeline_templates().create.refute_called()
1717         stubs.api.container_requests().create.refute_called()
1718
1719         body = {
1720             "workflow": {
1721                 "owner_uuid": project_uuid,
1722                 "name": "testing 123",
1723                 "description": "",
1724                 "definition": self.expect_workflow_altname,
1725             }
1726         }
1727         stubs.api.workflows().create.assert_called_with(
1728             body=JsonDiffMatcher(body))
1729
1730         self.assertEqual(stubs.capture_stdout.getvalue(),
1731                          stubs.expect_workflow_uuid + '\n')
1732         self.assertEqual(exited, 0)
1733
1734
1735     @stubs()
1736     def test_update(self, stubs):
1737         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1738         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1739
1740         exited = arvados_cwl.main(
1741             ["--update-workflow", self.existing_workflow_uuid,
1742              "--debug",
1743              "--disable-git",
1744              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1745             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1746
1747         body = {
1748             "workflow": {
1749                 "name": "submit_wf.cwl",
1750                 "description": "",
1751                 "definition": self.expect_workflow,
1752                 "owner_uuid": project_uuid
1753             }
1754         }
1755         stubs.api.workflows().update.assert_called_with(
1756             uuid=self.existing_workflow_uuid,
1757             body=JsonDiffMatcher(body))
1758         self.assertEqual(stubs.capture_stdout.getvalue(),
1759                          self.existing_workflow_uuid + '\n')
1760         self.assertEqual(exited, 0)
1761
1762
1763     @stubs()
1764     def test_update_name(self, stubs):
1765         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1766         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1767
1768         exited = arvados_cwl.main(
1769             ["--update-workflow", self.existing_workflow_uuid,
1770              "--debug", "--name", "testing 123",
1771              "--disable-git",
1772              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1773             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1774
1775         body = {
1776             "workflow": {
1777                 "name": "testing 123",
1778                 "description": "",
1779                 "definition": self.expect_workflow_altname,
1780                 "owner_uuid": project_uuid
1781             }
1782         }
1783         stubs.api.workflows().update.assert_called_with(
1784             uuid=self.existing_workflow_uuid,
1785             body=JsonDiffMatcher(body))
1786         self.assertEqual(stubs.capture_stdout.getvalue(),
1787                          self.existing_workflow_uuid + '\n')
1788         self.assertEqual(exited, 0)
1789
1790     @stubs()
1791     def test_create_collection_per_tool(self, stubs):
1792         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1793         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1794
1795         exited = arvados_cwl.main(
1796             ["--create-workflow", "--debug",
1797              "--api=containers",
1798              "--project-uuid", project_uuid,
1799              "--disable-git",
1800              "tests/collection_per_tool/collection_per_tool.cwl"],
1801             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1802
1803         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1804         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1805
1806         body = {
1807             "workflow": {
1808                 "owner_uuid": project_uuid,
1809                 "name": "collection_per_tool.cwl",
1810                 "description": "",
1811                 "definition": expect_workflow,
1812             }
1813         }
1814         stubs.api.workflows().create.assert_called_with(
1815             body=JsonDiffMatcher(body))
1816
1817         self.assertEqual(stubs.capture_stdout.getvalue(),
1818                          stubs.expect_workflow_uuid + '\n')
1819         self.assertEqual(exited, 0)
1820
1821     @stubs()
1822     def test_create_with_imports(self, stubs):
1823         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1824         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1825
1826         exited = arvados_cwl.main(
1827             ["--create-workflow", "--debug",
1828              "--api=containers",
1829              "--project-uuid", project_uuid,
1830              "tests/wf/feddemo/feddemo.cwl"],
1831             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1832
1833         stubs.api.pipeline_templates().create.refute_called()
1834         stubs.api.container_requests().create.refute_called()
1835
1836         self.assertEqual(stubs.capture_stdout.getvalue(),
1837                          stubs.expect_workflow_uuid + '\n')
1838         self.assertEqual(exited, 0)
1839
1840     @stubs()
1841     def test_create_with_no_input(self, stubs):
1842         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1843         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1844
1845         exited = arvados_cwl.main(
1846             ["--create-workflow", "--debug",
1847              "--api=containers",
1848              "--project-uuid", project_uuid,
1849              "tests/wf/revsort/revsort.cwl"],
1850             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1851
1852         stubs.api.pipeline_templates().create.refute_called()
1853         stubs.api.container_requests().create.refute_called()
1854
1855         self.assertEqual(stubs.capture_stdout.getvalue(),
1856                          stubs.expect_workflow_uuid + '\n')
1857         self.assertEqual(exited, 0)
1858
1859     @stubs()
1860     def test_create_map(self, stubs):
1861         # test uploading a document that uses objects instead of arrays
1862         # for certain fields like inputs and requirements.
1863
1864         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1865         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1866
1867         exited = arvados_cwl.main(
1868             ["--create-workflow", "--debug",
1869              "--api=containers",
1870              "--project-uuid", project_uuid,
1871              "--disable-git",
1872              "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
1873             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1874
1875         stubs.api.pipeline_templates().create.refute_called()
1876         stubs.api.container_requests().create.refute_called()
1877
1878         expect_workflow = StripYAMLComments(
1879             open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
1880
1881         body = {
1882             "workflow": {
1883                 "owner_uuid": project_uuid,
1884                 "name": "submit_wf_map.cwl",
1885                 "description": "",
1886                 "definition": expect_workflow,
1887             }
1888         }
1889         stubs.api.workflows().create.assert_called_with(
1890             body=JsonDiffMatcher(body))
1891
1892         self.assertEqual(stubs.capture_stdout.getvalue(),
1893                          stubs.expect_workflow_uuid + '\n')
1894         self.assertEqual(exited, 0)
1895
1896
1897 class TestPrintKeepDeps(unittest.TestCase):
1898     @stubs()
1899     def test_print_keep_deps(self, stubs):
1900         # test --print-keep-deps which is used by arv-copy
1901
1902         exited = arvados_cwl.main(
1903             ["--print-keep-deps", "--debug",
1904              "tests/wf/submit_wf_map.cwl"],
1905             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1906
1907         self.assertEqual(stubs.capture_stdout.getvalue(),
1908                          '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
1909         self.assertEqual(exited, 0)