sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import copy
   6 import io
   7 import itertools
   8 import functools
   9 import hashlib
  10 import json
  11 import logging
  12 import sys
  13 import unittest
  14 import cwltool.process
  15 import re
  16 import os
  17
  18 from io import BytesIO, StringIO
  19 from unittest import mock
  20
  21 import arvados
  22 import arvados.collection
  23 import arvados_cwl
  24 import arvados_cwl.executor
  25 import arvados_cwl.runner
  26 import arvados.keep
  27
  28 from .matcher import JsonDiffMatcher, StripYAMLComments
  29 from .mock_discovery import get_rootDesc
  30
  31 import ruamel.yaml
  32
  33 _rootDesc = None
  34
  35 def stubs(wfdetails=('submit_wf.cwl', None)):
  36     def outer_wrapper(func, *rest):
  37         @functools.wraps(func)
  38         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  39         @mock.patch("uuid.uuid4")
  40         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  41         @mock.patch("arvados.collection.KeepClient")
  42         @mock.patch("arvados.keep.KeepClient")
  43         @mock.patch("arvados.events.subscribe")
  44         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  45                     uuid4, determine_image_id, *args, **kwargs):
  46             class Stubs(object):
  47                 pass
  48
  49             wfname = wfdetails[0]
  50             wfpath = wfdetails[1]
  51
  52             stubs = Stubs()
  53             stubs.events = events
  54             stubs.keepdocker = keepdocker
  55
  56             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  57                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  58                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  59                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  60                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  61                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  62                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  63                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  64                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  65
  66             determine_image_id.return_value = None
  67
  68             def putstub(p, **kwargs):
  69                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  70             keep_client1().put.side_effect = putstub
  71             keep_client1.put.side_effect = putstub
  72             keep_client2().put.side_effect = putstub
  73             keep_client2.put.side_effect = putstub
  74
  75             stubs.keep_client = keep_client2
  76             stubs.docker_images = {
  77                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  78                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  79                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  80                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  81             }
  82             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  83                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  84             stubs.keepdocker.side_effect = kd
  85
  86             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  87             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  88
  89             if sys.version_info[0] < 3:
  90                 stubs.capture_stdout = BytesIO()
  91             else:
  92                 stubs.capture_stdout = StringIO()
  93
  94             stubs.api = mock.MagicMock()
  95             stubs.api._rootDesc = get_rootDesc()
  96             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
  97             stubs.api._rootDesc["revision"] = "20210628"
  98
  99             stubs.api.users().current().execute.return_value = {
 100                 "uuid": stubs.fake_user_uuid,
 101             }
 102             stubs.api.collections().list().execute.return_value = {"items": []}
 103             stubs.api.containers().current().execute.return_value = {
 104                 "uuid": stubs.fake_container_uuid,
 105             }
 106             stubs.api.config()["StorageClasses"].items.return_value = {
 107                 "default": {
 108                     "Default": True
 109                 }
 110             }.items()
 111
 112             class CollectionExecute(object):
 113                 def __init__(self, exe):
 114                     self.exe = exe
 115                 def execute(self, num_retries=None):
 116                     return self.exe
 117
 118             def collection_createstub(created_collections, body, ensure_unique_name=None):
 119                 mt = body["manifest_text"].encode('utf-8')
 120                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 121                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 122                 created_collections[uuid] = {
 123                     "uuid": uuid,
 124                     "portable_data_hash": pdh,
 125                     "manifest_text": mt.decode('utf-8')
 126                 }
 127                 return CollectionExecute(created_collections[uuid])
 128
 129             def collection_getstub(created_collections, uuid):
 130                 for v in created_collections.values():
 131                     if uuid in (v["uuid"], v["portable_data_hash"]):
 132                         return CollectionExecute(v)
 133
 134             created_collections = {
 135                 "99999999999999999999999999999998+99": {
 136                     "uuid": "",
 137                     "portable_data_hash": "99999999999999999999999999999998+99",
 138                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 139                 },
 140                 "99999999999999999999999999999997+99": {
 141                     "uuid": "",
 142                     "portable_data_hash": "99999999999999999999999999999997+99",
 143                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 144                 },
 145                 "99999999999999999999999999999994+99": {
 146                     "uuid": "",
 147                     "portable_data_hash": "99999999999999999999999999999994+99",
 148                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 149                 },
 150                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 151                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 152                     "portable_data_hash": "999999999999999999999999999999d3+99",
 153                     "manifest_text": ""
 154                 },
 155                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 156                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 157                     "portable_data_hash": "999999999999999999999999999999d4+99",
 158                     "manifest_text": ""
 159                 },
 160                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 161                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 162                     "portable_data_hash": "999999999999999999999999999999d5+99",
 163                     "manifest_text": ""
 164                 },
 165                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 166                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 167                     "portable_data_hash": "999999999999999999999999999999d6+99",
 168                     "manifest_text": ""
 169                 }
 170             }
 171             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 172             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 173
 174             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 175             stubs.api.container_requests().create().execute.return_value = {
 176                 "uuid": stubs.expect_container_request_uuid,
 177                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 178                 "state": "Queued"
 179             }
 180
 181             cwd = os.getcwd()
 182             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 183             with open(filepath) as f:
 184                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 185                 expect_packed_workflow = yaml.load(f)
 186
 187             if wfpath is None:
 188                 wfpath = wfname
 189
 190             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 191             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 192             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 193
 194             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 195             expect_packed_workflow.update(stubs.git_info)
 196
 197             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 198
 199             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 200             if wfname == wfpath:
 201                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 202             else:
 203                 container_name = wfname
 204
 205             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 206             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 207
 208             stubs.expect_container_spec = {
 209                 'priority': 500,
 210                 'mounts': {
 211                     '/var/spool/cwl': {
 212                         'writable': True,
 213                         'kind': 'collection'
 214                     },
 215                     '/var/lib/cwl/workflow.json': {
 216                         'content': expect_packed_workflow,
 217                         'kind': 'json'
 218                     },
 219                     'stdout': {
 220                         'path': '/var/spool/cwl/cwl.output.json',
 221                         'kind': 'file'
 222                     },
 223                     '/var/lib/cwl/cwl.input.json': {
 224                         'kind': 'json',
 225                         'content': {
 226                             'y': {
 227                                 'basename': '99999999999999999999999999999998+99',
 228                                 'location': 'keep:99999999999999999999999999999998+99',
 229                                 'class': 'Directory'},
 230                             'x': {
 231                                 'basename': u'blorp.txt',
 232                                 'class': 'File',
 233                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 234                                 "size": 16
 235                             },
 236                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 237                                 {'basename': 'renamed.txt',
 238                                  'class': 'File',
 239                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 240                                  'size': 0
 241                                 }
 242                             ]}
 243                         },
 244                         'kind': 'json'
 245                     }
 246                 },
 247                 'secret_mounts': {},
 248                 'state': 'Committed',
 249                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 250                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 251                             '--eval-timeout=20', '--thread-count=0',
 252                             '--enable-reuse', "--collection-cache-size=256",
 253                             '--output-name=Output from workflow '+container_name,
 254                             '--debug', '--on-error=continue',
 255                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 256                 'name': container_name,
 257                 'container_image': '999999999999999999999999999999d3+99',
 258                 'output_name': 'Output from workflow %s' % (container_name),
 259                 'output_path': '/var/spool/cwl',
 260                 'cwd': '/var/spool/cwl',
 261                 'runtime_constraints': {
 262                     'API': True,
 263                     'vcpus': 1,
 264                     'ram': (1024+256)*1024*1024
 265                 },
 266                 'properties': stubs.git_props,
 267                 'use_existing': False,
 268                 'secret_mounts': {}
 269             }
 270
 271             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 272             stubs.api.workflows().create().execute.return_value = {
 273                 "uuid": stubs.expect_workflow_uuid,
 274             }
 275             def update_mock(**kwargs):
 276                 stubs.updated_uuid = kwargs.get('uuid')
 277                 return mock.DEFAULT
 278             stubs.api.workflows().update.side_effect = update_mock
 279             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 280                 "uuid": stubs.updated_uuid,
 281             }
 282
 283             return func(self, stubs, *args, **kwargs)
 284         return wrapped
 285     return outer_wrapper
 286
 287 class TestSubmit(unittest.TestCase):
 288
 289     def setUp(self):
 290         cwltool.process._names = set()
 291         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 292
 293     def tearDown(self):
 294         root_logger = logging.getLogger('')
 295
 296         # Remove existing RuntimeStatusLoggingHandlers if they exist
 297         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 298         root_logger.handlers = handlers
 299
 300     @mock.patch("time.sleep")
 301     @stubs()
 302     def test_submit_invalid_runner_ram(self, stubs, tm):
 303         exited = arvados_cwl.main(
 304             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 305              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 306             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 307         self.assertEqual(exited, 1)
 308
 309
 310     @stubs()
 311     def test_submit_container(self, stubs):
 312         exited = arvados_cwl.main(
 313             ["--submit", "--no-wait", "--api=containers", "--debug",
 314                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 316
 317         stubs.api.collections().create.assert_has_calls([
 318             mock.call(body=JsonDiffMatcher({
 319                 'manifest_text':
 320                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 321                 'replication_desired': None,
 322                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 323             }), ensure_unique_name=False),
 324             mock.call(body=JsonDiffMatcher({
 325                 'manifest_text':
 326                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 327                 'replication_desired': None,
 328                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 329             }), ensure_unique_name=False),
 330             ])
 331
 332         expect_container = copy.deepcopy(stubs.expect_container_spec)
 333         stubs.api.container_requests().create.assert_called_with(
 334             body=JsonDiffMatcher(expect_container))
 335         self.assertEqual(stubs.capture_stdout.getvalue(),
 336                          stubs.expect_container_request_uuid + '\n')
 337         self.assertEqual(exited, 0)
 338
 339
 340     @stubs()
 341     def test_submit_container_tool(self, stubs):
 342         # test for issue #16139
 343         exited = arvados_cwl.main(
 344             ["--submit", "--no-wait", "--api=containers", "--debug",
 345                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 346             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 347
 348         self.assertEqual(stubs.capture_stdout.getvalue(),
 349                          stubs.expect_container_request_uuid + '\n')
 350         self.assertEqual(exited, 0)
 351
 352     @stubs()
 353     def test_submit_container_no_reuse(self, stubs):
 354         exited = arvados_cwl.main(
 355             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 356                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 358
 359         expect_container = copy.deepcopy(stubs.expect_container_spec)
 360         expect_container["command"] = [
 361             'arvados-cwl-runner', '--local', '--api=containers',
 362             '--no-log-timestamps', '--disable-validate', '--disable-color',
 363             '--eval-timeout=20', '--thread-count=0',
 364             '--disable-reuse', "--collection-cache-size=256",
 365             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 366             '--debug', '--on-error=continue',
 367             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 368         expect_container["use_existing"] = False
 369
 370         stubs.api.container_requests().create.assert_called_with(
 371             body=JsonDiffMatcher(expect_container))
 372         self.assertEqual(stubs.capture_stdout.getvalue(),
 373                          stubs.expect_container_request_uuid + '\n')
 374         self.assertEqual(exited, 0)
 375
 376     @stubs(('submit_wf_no_reuse.cwl', None))
 377     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 378         exited = arvados_cwl.main(
 379             ["--submit", "--no-wait", "--api=containers", "--debug",
 380              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 381             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 382         self.assertEqual(exited, 0)
 383
 384         expect_container = copy.deepcopy(stubs.expect_container_spec)
 385         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 386         expect_container["use_existing"] = False
 387         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 388             {
 389                 "class": "WorkReuse",
 390                 "enableReuse": False,
 391             },
 392                 {
 393                     "acrContainerImage": "999999999999999999999999999999d3+99",
 394                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 395                 }
 396         ]
 397         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 398
 399         stubs.api.container_requests().create.assert_called_with(
 400             body=JsonDiffMatcher(expect_container))
 401         self.assertEqual(stubs.capture_stdout.getvalue(),
 402                          stubs.expect_container_request_uuid + '\n')
 403
 404
 405     @stubs()
 406     def test_submit_container_on_error(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         expect_container = copy.deepcopy(stubs.expect_container_spec)
 413         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 414                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 415                                        '--eval-timeout=20', '--thread-count=0',
 416                                        '--enable-reuse', "--collection-cache-size=256",
 417                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 418                                        '--debug', '--on-error=stop',
 419                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 420
 421         stubs.api.container_requests().create.assert_called_with(
 422             body=JsonDiffMatcher(expect_container))
 423         self.assertEqual(stubs.capture_stdout.getvalue(),
 424                          stubs.expect_container_request_uuid + '\n')
 425         self.assertEqual(exited, 0)
 426
 427     @stubs()
 428     def test_submit_container_output_name(self, stubs):
 429         output_name = "test_output_name"
 430
 431         exited = arvados_cwl.main(
 432             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 433                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 435
 436         expect_container = copy.deepcopy(stubs.expect_container_spec)
 437         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 438                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 439                                        '--eval-timeout=20', '--thread-count=0',
 440                                        '--enable-reuse', "--collection-cache-size=256",
 441                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 442                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 443         expect_container["output_name"] = output_name
 444
 445         stubs.api.container_requests().create.assert_called_with(
 446             body=JsonDiffMatcher(expect_container))
 447         self.assertEqual(stubs.capture_stdout.getvalue(),
 448                          stubs.expect_container_request_uuid + '\n')
 449         self.assertEqual(exited, 0)
 450
 451     @stubs()
 452     def test_submit_storage_classes(self, stubs):
 453         exited = arvados_cwl.main(
 454             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 455                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 456             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 457
 458         expect_container = copy.deepcopy(stubs.expect_container_spec)
 459         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 460                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 461                                        '--eval-timeout=20', '--thread-count=0',
 462                                        '--enable-reuse', "--collection-cache-size=256",
 463                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 464                                        "--debug",
 465                                        "--storage-classes=foo", '--on-error=continue',
 466                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 467
 468         stubs.api.container_requests().create.assert_called_with(
 469             body=JsonDiffMatcher(expect_container))
 470         self.assertEqual(stubs.capture_stdout.getvalue(),
 471                          stubs.expect_container_request_uuid + '\n')
 472         self.assertEqual(exited, 0)
 473
 474     @stubs()
 475     def test_submit_multiple_storage_classes(self, stubs):
 476         exited = arvados_cwl.main(
 477             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 478                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 479             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 480
 481         expect_container = copy.deepcopy(stubs.expect_container_spec)
 482         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 483                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 484                                        '--eval-timeout=20', '--thread-count=0',
 485                                        '--enable-reuse', "--collection-cache-size=256",
 486                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 487                                        "--debug",
 488                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 489                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 490
 491         stubs.api.container_requests().create.assert_called_with(
 492             body=JsonDiffMatcher(expect_container))
 493         self.assertEqual(stubs.capture_stdout.getvalue(),
 494                          stubs.expect_container_request_uuid + '\n')
 495         self.assertEqual(exited, 0)
 496
 497     @mock.patch("cwltool.task_queue.TaskQueue")
 498     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 499     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 500     @stubs()
 501     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 502         final_output_c = arvados.collection.Collection()
 503         make_output.return_value = ({},final_output_c)
 504
 505         def set_final_output(job_order, output_callback, runtimeContext):
 506             output_callback({"out": "zzzzz"}, "success")
 507             return []
 508         job.side_effect = set_final_output
 509
 510         exited = arvados_cwl.main(
 511             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 512                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514
 515         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 516         self.assertEqual(exited, 0)
 517
 518     @mock.patch("cwltool.task_queue.TaskQueue")
 519     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 520     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 521     @stubs()
 522     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 523         final_output_c = arvados.collection.Collection()
 524         make_output.return_value = ({},final_output_c)
 525         stubs.api.config().get.return_value = {"default": {"Default": True}}
 526
 527         def set_final_output(job_order, output_callback, runtimeContext):
 528             output_callback({"out": "zzzzz"}, "success")
 529             return []
 530         job.side_effect = set_final_output
 531
 532         exited = arvados_cwl.main(
 533             ["--debug", "--local", "--disable-git",
 534                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 535             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 536
 537         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 538         self.assertEqual(exited, 0)
 539
 540     @mock.patch("cwltool.task_queue.TaskQueue")
 541     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 542     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 543     @stubs()
 544     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 545         final_output_c = arvados.collection.Collection()
 546         make_output.return_value = ({},final_output_c)
 547
 548         def set_final_output(job_order, output_callback, runtimeContext):
 549             output_callback({"out": "zzzzz"}, "success")
 550             return []
 551         job.side_effect = set_final_output
 552
 553         exited = arvados_cwl.main(
 554             ["--debug", "--local", "--disable-git",
 555                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 556             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 557
 558         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 559         self.assertEqual(exited, 0)
 560
 561     @stubs()
 562     def test_submit_container_output_ttl(self, stubs):
 563         exited = arvados_cwl.main(
 564             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 565                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 566             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 567
 568         expect_container = copy.deepcopy(stubs.expect_container_spec)
 569         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 570                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 571                                        '--eval-timeout=20', '--thread-count=0',
 572                                        '--enable-reuse', "--collection-cache-size=256",
 573                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 574                                        '--debug',
 575                                        '--on-error=continue',
 576                                        "--intermediate-output-ttl=3600",
 577                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 578
 579         stubs.api.container_requests().create.assert_called_with(
 580             body=JsonDiffMatcher(expect_container))
 581         self.assertEqual(stubs.capture_stdout.getvalue(),
 582                          stubs.expect_container_request_uuid + '\n')
 583         self.assertEqual(exited, 0)
 584
 585     @stubs()
 586     def test_submit_container_trash_intermediate(self, stubs):
 587         exited = arvados_cwl.main(
 588             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 589                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 590             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 591
 592
 593         expect_container = copy.deepcopy(stubs.expect_container_spec)
 594         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 595                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 596                                        '--eval-timeout=20', '--thread-count=0',
 597                                        '--enable-reuse', "--collection-cache-size=256",
 598                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 599                                        '--debug', '--on-error=continue',
 600                                        "--trash-intermediate",
 601                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 602
 603         stubs.api.container_requests().create.assert_called_with(
 604             body=JsonDiffMatcher(expect_container))
 605         self.assertEqual(stubs.capture_stdout.getvalue(),
 606                          stubs.expect_container_request_uuid + '\n')
 607         self.assertEqual(exited, 0)
 608
 609     @stubs()
 610     def test_submit_container_output_tags(self, stubs):
 611         output_tags = "tag0,tag1,tag2"
 612
 613         exited = arvados_cwl.main(
 614             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 615                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 616             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 617
 618         expect_container = copy.deepcopy(stubs.expect_container_spec)
 619         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 620                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 621                                        '--eval-timeout=20', '--thread-count=0',
 622                                        '--enable-reuse', "--collection-cache-size=256",
 623                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 624                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 625                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 626
 627         stubs.api.container_requests().create.assert_called_with(
 628             body=JsonDiffMatcher(expect_container))
 629         self.assertEqual(stubs.capture_stdout.getvalue(),
 630                          stubs.expect_container_request_uuid + '\n')
 631         self.assertEqual(exited, 0)
 632
 633     @stubs()
 634     def test_submit_container_runner_ram(self, stubs):
 635         exited = arvados_cwl.main(
 636             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 637                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 638             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 639
 640         expect_container = copy.deepcopy(stubs.expect_container_spec)
 641         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 642         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 643                 {
 644                     "acrContainerImage": "999999999999999999999999999999d3+99",
 645                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 646                     "ramMin": 2048
 647                 }
 648         ]
 649
 650         stubs.api.container_requests().create.assert_called_with(
 651             body=JsonDiffMatcher(expect_container))
 652         self.assertEqual(stubs.capture_stdout.getvalue(),
 653                          stubs.expect_container_request_uuid + '\n')
 654         self.assertEqual(exited, 0)
 655
 656     @mock.patch("arvados.collection.CollectionReader")
 657     @mock.patch("time.sleep")
 658     @stubs()
 659     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 660         collectionReader().exists.return_value = True
 661         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 662         exited = arvados_cwl.main(
 663             ["--submit", "--no-wait", "--api=containers", "--debug",
 664              "tests/wf/submit_keepref_wf.cwl"],
 665             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 666         self.assertEqual(exited, 0)
 667
 668     @mock.patch("arvados.collection.CollectionReader")
 669     @mock.patch("time.sleep")
 670     @stubs()
 671     def test_submit_keepref(self, stubs, tm, reader):
 672         with open("tests/wf/expect_arvworkflow.cwl") as f:
 673             reader().open().__enter__().read.return_value = f.read()
 674
 675         exited = arvados_cwl.main(
 676             ["--submit", "--no-wait", "--api=containers", "--debug",
 677              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 678             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 679
 680         expect_container = {
 681             'priority': 500,
 682             'mounts': {
 683                 '/var/spool/cwl': {
 684                     'writable': True,
 685                     'kind': 'collection'
 686                 },
 687                 'stdout': {
 688                     'path': '/var/spool/cwl/cwl.output.json',
 689                     'kind': 'file'
 690                 },
 691                 '/var/lib/cwl/workflow': {
 692                     'portable_data_hash': '99999999999999999999999999999994+99',
 693                     'kind': 'collection'
 694                 },
 695                 '/var/lib/cwl/cwl.input.json': {
 696                     'content': {
 697                         'x': 'XxX'
 698                     },
 699                     'kind': 'json'
 700                 }
 701             }, 'state': 'Committed',
 702             'output_path': '/var/spool/cwl',
 703             'name': 'expect_arvworkflow.cwl#main',
 704             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 705             'container_image': '999999999999999999999999999999d3+99',
 706             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 707                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 708                         '--eval-timeout=20', '--thread-count=0',
 709                         '--enable-reuse', "--collection-cache-size=256",
 710                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 711                         '--debug', '--on-error=continue',
 712                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 713             'cwd': '/var/spool/cwl',
 714             'runtime_constraints': {
 715                 'API': True,
 716                 'vcpus': 1,
 717                 'ram': 1342177280
 718             },
 719             'use_existing': False,
 720             'properties': {},
 721             'secret_mounts': {}
 722         }
 723
 724         stubs.api.container_requests().create.assert_called_with(
 725             body=JsonDiffMatcher(expect_container))
 726         self.assertEqual(stubs.capture_stdout.getvalue(),
 727                          stubs.expect_container_request_uuid + '\n')
 728         self.assertEqual(exited, 0)
 729
 730     @mock.patch("time.sleep")
 731     @stubs()
 732     def test_submit_arvworkflow(self, stubs, tm):
 733         with open("tests/wf/expect_arvworkflow.cwl") as f:
 734             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 735
 736         exited = arvados_cwl.main(
 737             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 738              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 739             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 740
 741         expect_container = {
 742             'priority': 500,
 743             'mounts': {
 744                 '/var/spool/cwl': {
 745                     'writable': True,
 746                     'kind': 'collection'
 747                 },
 748                 'stdout': {
 749                     'path': '/var/spool/cwl/cwl.output.json',
 750                     'kind': 'file'
 751                 },
 752                 '/var/lib/cwl/workflow.json': {
 753                     'kind': 'json',
 754                     'content': {
 755                         'cwlVersion': 'v1.0',
 756                         'label': 'a test workflow',
 757                         '$graph': [
 758                             {
 759                                 'id': '#main',
 760                                 'inputs': [
 761                                     {'type': 'string', 'id': '#main/x'}
 762                                 ],
 763                                 'steps': [
 764                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 765                                      'run': '#submit_tool.cwl',
 766                                      'id': '#main/step1',
 767                                      'out': []}
 768                                 ],
 769                                 'class': 'Workflow',
 770                                 'outputs': []
 771                             },
 772                             {
 773                                 'inputs': [
 774                                     {
 775                                         'inputBinding': {'position': 1},
 776                                         'type': 'string',
 777                                         'id': '#submit_tool.cwl/x'}
 778                                 ],
 779                                 'requirements': [
 780                                     {
 781                                         'dockerPull': 'debian:buster-slim',
 782                                         'class': 'DockerRequirement'
 783                                     }
 784                                 ],
 785                                 'id': '#submit_tool.cwl',
 786                                 'outputs': [],
 787                                 'baseCommand': 'cat',
 788                                 'class': 'CommandLineTool'
 789                             }
 790                         ]
 791                     }
 792                 },
 793                 '/var/lib/cwl/cwl.input.json': {
 794                     'content': {
 795                         'x': 'XxX'
 796                     },
 797                     'kind': 'json'
 798                 }
 799             }, 'state': 'Committed',
 800             'output_path': '/var/spool/cwl',
 801             'name': 'a test workflow',
 802             'container_image': "999999999999999999999999999999d3+99",
 803             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 804                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 805                         '--eval-timeout=20', '--thread-count=0',
 806                         '--enable-reuse', "--collection-cache-size=256",
 807                         "--output-name=Output from workflow a test workflow",
 808                         '--debug', '--on-error=continue',
 809                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 810             'output_name': 'Output from workflow a test workflow',
 811             'cwd': '/var/spool/cwl',
 812             'runtime_constraints': {
 813                 'API': True,
 814                 'vcpus': 1,
 815                 'ram': 1342177280
 816             },
 817             'use_existing': False,
 818             'properties': {
 819                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 820             },
 821             'secret_mounts': {}
 822         }
 823
 824         stubs.api.container_requests().create.assert_called_with(
 825             body=JsonDiffMatcher(expect_container))
 826         self.assertEqual(stubs.capture_stdout.getvalue(),
 827                          stubs.expect_container_request_uuid + '\n')
 828         self.assertEqual(exited, 0)
 829
 830     @stubs(('hello container 123', 'submit_wf.cwl'))
 831     def test_submit_container_name(self, stubs):
 832         exited = arvados_cwl.main(
 833             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 834                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 835             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 836
 837         expect_container = copy.deepcopy(stubs.expect_container_spec)
 838
 839         stubs.api.container_requests().create.assert_called_with(
 840             body=JsonDiffMatcher(expect_container))
 841         self.assertEqual(stubs.capture_stdout.getvalue(),
 842                          stubs.expect_container_request_uuid + '\n')
 843         self.assertEqual(exited, 0)
 844
 845     @stubs()
 846     def test_submit_missing_input(self, stubs):
 847         exited = arvados_cwl.main(
 848             ["--submit", "--no-wait", "--api=containers", "--debug",
 849              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 850             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 851         self.assertEqual(exited, 0)
 852
 853         exited = arvados_cwl.main(
 854             ["--submit", "--no-wait", "--api=containers", "--debug",
 855              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 856             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 857         self.assertEqual(exited, 1)
 858
 859     @stubs()
 860     def test_submit_container_project(self, stubs):
 861         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 862         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 863         exited = arvados_cwl.main(
 864             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 865                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 866             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 867
 868         expect_container = copy.deepcopy(stubs.expect_container_spec)
 869         expect_container["owner_uuid"] = project_uuid
 870         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 871                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 872                                        "--eval-timeout=20", "--thread-count=0",
 873                                        '--enable-reuse', "--collection-cache-size=256",
 874                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 875                                        '--debug',
 876                                        '--on-error=continue',
 877                                        '--project-uuid='+project_uuid,
 878                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 879
 880         stubs.api.container_requests().create.assert_called_with(
 881             body=JsonDiffMatcher(expect_container))
 882         self.assertEqual(stubs.capture_stdout.getvalue(),
 883                          stubs.expect_container_request_uuid + '\n')
 884         self.assertEqual(exited, 0)
 885
 886     @stubs()
 887     def test_submit_container_eval_timeout(self, stubs):
 888         exited = arvados_cwl.main(
 889             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 890                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 891             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 892
 893         expect_container = copy.deepcopy(stubs.expect_container_spec)
 894         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 895                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 896                                        '--eval-timeout=60.0', '--thread-count=0',
 897                                        '--enable-reuse', "--collection-cache-size=256",
 898                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 899                                        '--debug', '--on-error=continue',
 900                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 901
 902         stubs.api.container_requests().create.assert_called_with(
 903             body=JsonDiffMatcher(expect_container))
 904         self.assertEqual(stubs.capture_stdout.getvalue(),
 905                          stubs.expect_container_request_uuid + '\n')
 906         self.assertEqual(exited, 0)
 907
 908     @stubs()
 909     def test_submit_container_collection_cache(self, stubs):
 910         exited = arvados_cwl.main(
 911             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 912                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 913             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 914
 915         expect_container = copy.deepcopy(stubs.expect_container_spec)
 916         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 917                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 918                                        '--eval-timeout=20', '--thread-count=0',
 919                                        '--enable-reuse', "--collection-cache-size=500",
 920                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 921                                        '--debug', '--on-error=continue',
 922                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 923         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 924
 925         stubs.api.container_requests().create.assert_called_with(
 926             body=JsonDiffMatcher(expect_container))
 927         self.assertEqual(stubs.capture_stdout.getvalue(),
 928                          stubs.expect_container_request_uuid + '\n')
 929         self.assertEqual(exited, 0)
 930
 931     @stubs()
 932     def test_submit_container_thread_count(self, stubs):
 933         exited = arvados_cwl.main(
 934             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 935                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 936             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 937
 938         expect_container = copy.deepcopy(stubs.expect_container_spec)
 939         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 940                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 941                                        '--eval-timeout=20', '--thread-count=20',
 942                                        '--enable-reuse', "--collection-cache-size=256",
 943                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 944                                        '--debug', '--on-error=continue',
 945                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 946
 947         stubs.api.container_requests().create.assert_called_with(
 948             body=JsonDiffMatcher(expect_container))
 949         self.assertEqual(stubs.capture_stdout.getvalue(),
 950                          stubs.expect_container_request_uuid + '\n')
 951         self.assertEqual(exited, 0)
 952
 953     @stubs()
 954     def test_submit_container_runner_image(self, stubs):
 955         exited = arvados_cwl.main(
 956             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 957                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 958             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 959
 960         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 961         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 962                 {
 963                     "acrContainerImage": "999999999999999999999999999999d5+99",
 964                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 965                 }
 966         ]
 967
 968         expect_container = copy.deepcopy(stubs.expect_container_spec)
 969         stubs.api.container_requests().create.assert_called_with(
 970             body=JsonDiffMatcher(expect_container))
 971         self.assertEqual(stubs.capture_stdout.getvalue(),
 972                          stubs.expect_container_request_uuid + '\n')
 973         self.assertEqual(exited, 0)
 974
 975     @stubs()
 976     def test_submit_priority(self, stubs):
 977         exited = arvados_cwl.main(
 978             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 979                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 980             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 981
 982         stubs.expect_container_spec["priority"] = 669
 983
 984         expect_container = copy.deepcopy(stubs.expect_container_spec)
 985         stubs.api.container_requests().create.assert_called_with(
 986             body=JsonDiffMatcher(expect_container))
 987         self.assertEqual(stubs.capture_stdout.getvalue(),
 988                          stubs.expect_container_request_uuid + '\n')
 989         self.assertEqual(exited, 0)
 990
 991     @stubs(('submit_wf_runner_resources.cwl', None))
 992     def test_submit_wf_runner_resources(self, stubs):
 993         exited = arvados_cwl.main(
 994             ["--submit", "--no-wait", "--api=containers", "--debug",
 995                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
 996             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 997
 998         expect_container = copy.deepcopy(stubs.expect_container_spec)
 999         expect_container["runtime_constraints"] = {
1000             "API": True,
1001             "vcpus": 2,
1002             "ram": (2000+512) * 2**20
1003         }
1004         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1005             {
1006                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1007                 "acrContainerImage": "999999999999999999999999999999d3+99",
1008                 "coresMin": 2,
1009                 "ramMin": 2000,
1010                 "keep_cache": 512
1011             }
1012         ]
1013         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1014         #    "arv": "http://arvados.org/cwl#",
1015         #}
1016         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1017         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1018
1019         stubs.api.container_requests().create.assert_called_with(
1020             body=JsonDiffMatcher(expect_container))
1021         self.assertEqual(stubs.capture_stdout.getvalue(),
1022                          stubs.expect_container_request_uuid + '\n')
1023         self.assertEqual(exited, 0)
1024
1025     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1026     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1027     @mock.patch("arvados.api")
1028     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1029         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1030
1031         arvrunner = mock.MagicMock()
1032         arvrunner.project_uuid = ""
1033         api.return_value = mock.MagicMock()
1034         arvrunner.api = api.return_value
1035         arvrunner.runtimeContext.match_local_docker = False
1036         arvrunner.api.links().list().execute.side_effect = itertools.cycle([
1037             {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
1038                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1039                         "link_class": "docker_image_repo+tag",
1040                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1041                         "owner_uuid": "",
1042                         "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
1043                         "properties": {"image_timestamp": ""}}]},
1044             {"items": []},
1045             {"items": []},
1046             {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
1047                         "head_uuid": "",
1048                         "link_class": "docker_image_hash",
1049                         "name": "123456",
1050                         "owner_uuid": "",
1051                         "uuid": "zzzzz-o0j2j-arvadosjobshash",
1052                         "properties": {"image_timestamp": ""}}]},
1053             {"items": []},
1054             {"items": []},
1055         ])
1056         find_one_image_hash.return_value = "123456"
1057
1058         arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
1059             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1060                         "owner_uuid": "",
1061                         "manifest_text": "",
1062                         "created_at": "2023-08-25T12:34:55.012345Z",
1063                         "properties": {}}]},
1064             {"items": []},
1065             {"items": []},
1066         ])
1067         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1068         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1069                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1070
1071         self.assertEqual("9999999999999999999999999999999b+99",
1072                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1073
1074
1075     @stubs()
1076     def test_submit_secrets(self, stubs):
1077         exited = arvados_cwl.main(
1078             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1079                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1080             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1081
1082         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1083         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1084
1085         expect_container = {
1086             "command": [
1087                 "arvados-cwl-runner",
1088                 "--local",
1089                 "--api=containers",
1090                 "--no-log-timestamps",
1091                 "--disable-validate",
1092                 "--disable-color",
1093                 "--eval-timeout=20",
1094                 '--thread-count=0',
1095                 "--enable-reuse",
1096                 "--collection-cache-size=256",
1097                 "--output-name=Output from workflow secret_wf.cwl",
1098                 "--debug",
1099                 "--on-error=continue",
1100                 "/var/lib/cwl/workflow.json#main",
1101                 "/var/lib/cwl/cwl.input.json"
1102             ],
1103             "container_image": "999999999999999999999999999999d3+99",
1104             "cwd": "/var/spool/cwl",
1105             "mounts": {
1106                 "/var/lib/cwl/cwl.input.json": {
1107                     "content": {
1108                         "pw": {
1109                             "$include": "/secrets/s0"
1110                         }
1111                     },
1112                     "kind": "json"
1113                 },
1114                 "/var/lib/cwl/workflow.json": {
1115                     "content": {
1116                         "$graph": [
1117                             {
1118                                 "class": "Workflow",
1119                                 "hints": [
1120                                     {
1121                                     "class": "DockerRequirement",
1122                                     "dockerPull": "debian:buster-slim",
1123                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1124                                     },
1125                                     {
1126                                         "class": "http://commonwl.org/cwltool#Secrets",
1127                                         "secrets": [
1128                                             "#main/pw"
1129                                         ]
1130                                     },
1131                                     {
1132                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1133                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1134                                     }
1135                                 ],
1136                                 "id": "#main",
1137                                 "inputs": [
1138                                     {
1139                                         "id": "#main/pw",
1140                                         "type": "string"
1141                                     }
1142                                 ],
1143                                 "outputs": [
1144                                     {
1145                                         "id": "#main/out",
1146                                         "outputSource": "#main/step/out",
1147                                         "type": "File"
1148                                     }
1149                                 ],
1150                                 "requirements": [
1151                                     {
1152                                         "class": "SubworkflowFeatureRequirement"
1153                                     }
1154                                 ],
1155                                 "steps": [
1156                                     {
1157                                         "id": "#main/secret_wf.cwl",
1158                                         "in": [
1159                                             {
1160                                                 "id": "#main/step/pw",
1161                                                 "source": "#main/pw"
1162                                             }
1163                                         ],
1164                                         "label": "secret_wf.cwl",
1165                                         "out": [
1166                                             {"id": "#main/step/out"}
1167                                         ],
1168                                         "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
1169                                     }
1170                                 ]
1171                             }
1172                         ],
1173                         "cwlVersion": "v1.2"
1174                     },
1175                     "kind": "json"
1176                 },
1177                 "/var/spool/cwl": {
1178                     "kind": "collection",
1179                     "writable": True
1180                 },
1181                 "stdout": {
1182                     "kind": "file",
1183                     "path": "/var/spool/cwl/cwl.output.json"
1184                 }
1185             },
1186             "name": "secret_wf.cwl",
1187             "output_name": "Output from workflow secret_wf.cwl",
1188             "output_path": "/var/spool/cwl",
1189             "priority": 500,
1190             "properties": {},
1191             "runtime_constraints": {
1192                 "API": True,
1193                 "ram": 1342177280,
1194                 "vcpus": 1
1195             },
1196             "secret_mounts": {
1197                 "/secrets/s0": {
1198                     "content": "blorp",
1199                     "kind": "text"
1200                 }
1201             },
1202             "state": "Committed",
1203             "use_existing": False
1204         }
1205
1206         stubs.api.container_requests().create.assert_called_with(
1207             body=JsonDiffMatcher(expect_container))
1208         self.assertEqual(stubs.capture_stdout.getvalue(),
1209                          stubs.expect_container_request_uuid + '\n')
1210         self.assertEqual(exited, 0)
1211
1212     @stubs()
1213     def test_submit_request_uuid(self, stubs):
1214         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1215         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1216
1217         stubs.api.container_requests().update().execute.return_value = {
1218             "uuid": stubs.expect_container_request_uuid,
1219             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1220             "state": "Queued"
1221         }
1222
1223         exited = arvados_cwl.main(
1224             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1225                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1226             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1227
1228         stubs.api.container_requests().update.assert_called_with(
1229             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1230         self.assertEqual(stubs.capture_stdout.getvalue(),
1231                          stubs.expect_container_request_uuid + '\n')
1232         self.assertEqual(exited, 0)
1233
1234     @stubs()
1235     def test_submit_container_cluster_id(self, stubs):
1236         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1237
1238         exited = arvados_cwl.main(
1239             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1240                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1241             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1242
1243         expect_container = copy.deepcopy(stubs.expect_container_spec)
1244
1245         stubs.api.container_requests().create.assert_called_with(
1246             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1247         self.assertEqual(stubs.capture_stdout.getvalue(),
1248                          stubs.expect_container_request_uuid + '\n')
1249         self.assertEqual(exited, 0)
1250
1251     @stubs()
1252     def test_submit_validate_cluster_id(self, stubs):
1253         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1254         exited = arvados_cwl.main(
1255             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1256              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1257             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1258         self.assertEqual(exited, 1)
1259
1260     @stubs()
1261     def test_submit_validate_project_uuid(self, stubs):
1262         # Fails with bad cluster prefix
1263         exited = arvados_cwl.main(
1264             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1265              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1266             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1267         self.assertEqual(exited, 1)
1268
1269         # Project lookup fails
1270         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1271         exited = arvados_cwl.main(
1272             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1273              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1274             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1275         self.assertEqual(exited, 1)
1276
1277         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1278         exited = arvados_cwl.main(
1279             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1280              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1281             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1282         self.assertEqual(exited, 0)
1283
1284
1285     @mock.patch("arvados.collection.CollectionReader")
1286     @stubs()
1287     def test_submit_uuid_inputs(self, stubs, collectionReader):
1288         collectionReader().exists.return_value = True
1289         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1290         def list_side_effect(**kwargs):
1291             m = mock.MagicMock()
1292             if "count" in kwargs:
1293                 m.execute.return_value = {"items": [
1294                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1295                 ]}
1296             else:
1297                 m.execute.return_value = {"items": []}
1298             return m
1299         stubs.api.collections().list.side_effect = list_side_effect
1300         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1301
1302         exited = arvados_cwl.main(
1303             ["--submit", "--no-wait", "--api=containers", "--debug",
1304                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1305             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1306
1307         expect_container = copy.deepcopy(stubs.expect_container_spec)
1308         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1309         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1310         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1311
1312         stubs.api.collections().list.assert_has_calls([
1313             mock.call(count='none',
1314                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1315                       select=['uuid', 'portable_data_hash'])])
1316         stubs.api.container_requests().create.assert_called_with(
1317             body=JsonDiffMatcher(expect_container))
1318         self.assertEqual(stubs.capture_stdout.getvalue(),
1319                          stubs.expect_container_request_uuid + '\n')
1320         self.assertEqual(exited, 0)
1321
1322     @stubs()
1323     def test_submit_mismatched_uuid_inputs(self, stubs):
1324         def list_side_effect(**kwargs):
1325             m = mock.MagicMock()
1326             if "count" in kwargs:
1327                 m.execute.return_value = {"items": [
1328                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1329                 ]}
1330             else:
1331                 m.execute.return_value = {"items": []}
1332             return m
1333         stubs.api.collections().list.side_effect = list_side_effect
1334
1335         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1336             capture_stderr = StringIO()
1337             cwltool_logger = logging.getLogger('cwltool')
1338             stderr_logger = logging.StreamHandler(capture_stderr)
1339             cwltool_logger.addHandler(stderr_logger)
1340
1341             try:
1342                 exited = arvados_cwl.main(
1343                     ["--submit", "--no-wait", "--api=containers", "--debug",
1344                         "tests/wf/submit_wf.cwl", infile],
1345                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1346
1347                 self.assertEqual(exited, 1)
1348                 self.assertRegex(
1349                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1350                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1351             finally:
1352                 cwltool_logger.removeHandler(stderr_logger)
1353
1354     @mock.patch("arvados.collection.CollectionReader")
1355     @stubs()
1356     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1357         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1358         capture_stderr = StringIO()
1359
1360         cwltool_logger = logging.getLogger('cwltool')
1361         stderr_logger = logging.StreamHandler(capture_stderr)
1362         cwltool_logger.addHandler(stderr_logger)
1363
1364         exited = arvados_cwl.main(
1365             ["--submit", "--no-wait", "--api=containers", "--debug",
1366                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1367             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1368
1369         try:
1370             self.assertEqual(exited, 1)
1371             self.assertRegex(
1372                 capture_stderr.getvalue(),
1373                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1374         finally:
1375             cwltool_logger.removeHandler(stderr_logger)
1376
1377     @stubs(('submit_wf_process_properties.cwl', None))
1378     def test_submit_set_process_properties(self, stubs):
1379         exited = arvados_cwl.main(
1380             ["--submit", "--no-wait", "--api=containers", "--debug",
1381                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1382             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1383
1384         expect_container = copy.deepcopy(stubs.expect_container_spec)
1385
1386         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1387             {
1388                 "class": "http://arvados.org/cwl#ProcessProperties",
1389                 "processProperties": [
1390                     {"propertyName": "baz",
1391                      "propertyValue": "$(inputs.x.basename)"},
1392                     {"propertyName": "foo",
1393                      "propertyValue": "bar"},
1394                     {"propertyName": "quux",
1395                      "propertyValue": {
1396                          "q1": 1,
1397                          "q2": 2
1398                      }
1399                     }
1400                 ],
1401             },
1402             {
1403                 "acrContainerImage": "999999999999999999999999999999d3+99",
1404                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1405             }
1406         ]
1407         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1408         #    "arv": "http://arvados.org/cwl#"
1409         #}
1410
1411         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1412
1413         expect_container["properties"].update({
1414             "baz": "blorp.txt",
1415             "foo": "bar",
1416             "quux": {
1417                 "q1": 1,
1418                 "q2": 2
1419             }
1420         })
1421
1422         stubs.api.container_requests().create.assert_called_with(
1423             body=JsonDiffMatcher(expect_container))
1424         self.assertEqual(stubs.capture_stdout.getvalue(),
1425                          stubs.expect_container_request_uuid + '\n')
1426         self.assertEqual(exited, 0)
1427
1428
1429     @stubs()
1430     def test_submit_enable_preemptible(self, stubs):
1431         exited = arvados_cwl.main(
1432             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1433                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1434             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1435
1436         expect_container = copy.deepcopy(stubs.expect_container_spec)
1437         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1438                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1439                                        '--eval-timeout=20', '--thread-count=0',
1440                                        '--enable-reuse', "--collection-cache-size=256",
1441                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1442                                        '--debug', '--on-error=continue',
1443                                        '--enable-preemptible',
1444                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1445
1446         stubs.api.container_requests().create.assert_called_with(
1447             body=JsonDiffMatcher(expect_container))
1448         self.assertEqual(stubs.capture_stdout.getvalue(),
1449                          stubs.expect_container_request_uuid + '\n')
1450         self.assertEqual(exited, 0)
1451
1452     @stubs()
1453     def test_submit_disable_preemptible(self, stubs):
1454         exited = arvados_cwl.main(
1455             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1456                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1457             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1458
1459         expect_container = copy.deepcopy(stubs.expect_container_spec)
1460         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1461                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1462                                        '--eval-timeout=20', '--thread-count=0',
1463                                        '--enable-reuse', "--collection-cache-size=256",
1464                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1465                                        '--debug', '--on-error=continue',
1466                                        '--disable-preemptible',
1467                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1468
1469         stubs.api.container_requests().create.assert_called_with(
1470             body=JsonDiffMatcher(expect_container))
1471         self.assertEqual(stubs.capture_stdout.getvalue(),
1472                          stubs.expect_container_request_uuid + '\n')
1473         self.assertEqual(exited, 0)
1474
1475     @stubs()
1476     def test_submit_container_prefer_cached_downloads(self, stubs):
1477         exited = arvados_cwl.main(
1478             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1479                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1480             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1481
1482         expect_container = copy.deepcopy(stubs.expect_container_spec)
1483         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1484                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1485                                        '--eval-timeout=20', '--thread-count=0',
1486                                        '--enable-reuse', "--collection-cache-size=256",
1487                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1488                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1489                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1490
1491         stubs.api.container_requests().create.assert_called_with(
1492             body=JsonDiffMatcher(expect_container))
1493         self.assertEqual(stubs.capture_stdout.getvalue(),
1494                          stubs.expect_container_request_uuid + '\n')
1495         self.assertEqual(exited, 0)
1496
1497     @stubs()
1498     def test_submit_container_varying_url_params(self, stubs):
1499         exited = arvados_cwl.main(
1500             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1501                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1502             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1503
1504         expect_container = copy.deepcopy(stubs.expect_container_spec)
1505         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1506                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1507                                        '--eval-timeout=20', '--thread-count=0',
1508                                        '--enable-reuse', "--collection-cache-size=256",
1509                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1510                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1511                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1512
1513         stubs.api.container_requests().create.assert_called_with(
1514             body=JsonDiffMatcher(expect_container))
1515         self.assertEqual(stubs.capture_stdout.getvalue(),
1516                          stubs.expect_container_request_uuid + '\n')
1517         self.assertEqual(exited, 0)
1518
1519
1520 class TestCreateWorkflow(unittest.TestCase):
1521     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1522     expect_workflow = StripYAMLComments(
1523         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1524     expect_workflow_altname = StripYAMLComments(
1525         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1526
1527     def setUp(self):
1528         cwltool.process._names = set()
1529         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1530
1531     def tearDown(self):
1532         root_logger = logging.getLogger('')
1533
1534         # Remove existing RuntimeStatusLoggingHandlers if they exist
1535         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1536         root_logger.handlers = handlers
1537
1538     @stubs()
1539     def test_create(self, stubs):
1540         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1541         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1542
1543         exited = arvados_cwl.main(
1544             ["--create-workflow", "--debug",
1545              "--api=containers",
1546              "--project-uuid", project_uuid,
1547              "--disable-git",
1548              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1549             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1550
1551         stubs.api.pipeline_templates().create.refute_called()
1552         stubs.api.container_requests().create.refute_called()
1553
1554         body = {
1555             "workflow": {
1556                 "owner_uuid": project_uuid,
1557                 "name": "submit_wf.cwl",
1558                 "description": "",
1559                 "definition": self.expect_workflow,
1560             }
1561         }
1562         stubs.api.workflows().create.assert_called_with(
1563             body=JsonDiffMatcher(body))
1564
1565         self.assertEqual(stubs.capture_stdout.getvalue(),
1566                          stubs.expect_workflow_uuid + '\n')
1567         self.assertEqual(exited, 0)
1568
1569     @stubs()
1570     def test_create_name(self, stubs):
1571         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1572         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1573
1574         exited = arvados_cwl.main(
1575             ["--create-workflow", "--debug",
1576              "--api=containers",
1577              "--project-uuid", project_uuid,
1578              "--name", "testing 123",
1579              "--disable-git",
1580              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1581             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1582
1583         stubs.api.pipeline_templates().create.refute_called()
1584         stubs.api.container_requests().create.refute_called()
1585
1586         body = {
1587             "workflow": {
1588                 "owner_uuid": project_uuid,
1589                 "name": "testing 123",
1590                 "description": "",
1591                 "definition": self.expect_workflow_altname,
1592             }
1593         }
1594         stubs.api.workflows().create.assert_called_with(
1595             body=JsonDiffMatcher(body))
1596
1597         self.assertEqual(stubs.capture_stdout.getvalue(),
1598                          stubs.expect_workflow_uuid + '\n')
1599         self.assertEqual(exited, 0)
1600
1601
1602     @stubs()
1603     def test_update(self, stubs):
1604         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1605         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1606
1607         exited = arvados_cwl.main(
1608             ["--update-workflow", self.existing_workflow_uuid,
1609              "--debug",
1610              "--disable-git",
1611              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1612             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1613
1614         body = {
1615             "workflow": {
1616                 "name": "submit_wf.cwl",
1617                 "description": "",
1618                 "definition": self.expect_workflow,
1619                 "owner_uuid": project_uuid
1620             }
1621         }
1622         stubs.api.workflows().update.assert_called_with(
1623             uuid=self.existing_workflow_uuid,
1624             body=JsonDiffMatcher(body))
1625         self.assertEqual(stubs.capture_stdout.getvalue(),
1626                          self.existing_workflow_uuid + '\n')
1627         self.assertEqual(exited, 0)
1628
1629
1630     @stubs()
1631     def test_update_name(self, stubs):
1632         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1633         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1634
1635         exited = arvados_cwl.main(
1636             ["--update-workflow", self.existing_workflow_uuid,
1637              "--debug", "--name", "testing 123",
1638              "--disable-git",
1639              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1640             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1641
1642         body = {
1643             "workflow": {
1644                 "name": "testing 123",
1645                 "description": "",
1646                 "definition": self.expect_workflow_altname,
1647                 "owner_uuid": project_uuid
1648             }
1649         }
1650         stubs.api.workflows().update.assert_called_with(
1651             uuid=self.existing_workflow_uuid,
1652             body=JsonDiffMatcher(body))
1653         self.assertEqual(stubs.capture_stdout.getvalue(),
1654                          self.existing_workflow_uuid + '\n')
1655         self.assertEqual(exited, 0)
1656
1657     @stubs()
1658     def test_create_collection_per_tool(self, stubs):
1659         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1660         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1661
1662         exited = arvados_cwl.main(
1663             ["--create-workflow", "--debug",
1664              "--api=containers",
1665              "--project-uuid", project_uuid,
1666              "--disable-git",
1667              "tests/collection_per_tool/collection_per_tool.cwl"],
1668             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1669
1670         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1671         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1672
1673         body = {
1674             "workflow": {
1675                 "owner_uuid": project_uuid,
1676                 "name": "collection_per_tool.cwl",
1677                 "description": "",
1678                 "definition": expect_workflow,
1679             }
1680         }
1681         stubs.api.workflows().create.assert_called_with(
1682             body=JsonDiffMatcher(body))
1683
1684         self.assertEqual(stubs.capture_stdout.getvalue(),
1685                          stubs.expect_workflow_uuid + '\n')
1686         self.assertEqual(exited, 0)
1687
1688     @stubs()
1689     def test_create_with_imports(self, stubs):
1690         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1691         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1692
1693         exited = arvados_cwl.main(
1694             ["--create-workflow", "--debug",
1695              "--api=containers",
1696              "--project-uuid", project_uuid,
1697              "tests/wf/feddemo/feddemo.cwl"],
1698             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1699
1700         stubs.api.pipeline_templates().create.refute_called()
1701         stubs.api.container_requests().create.refute_called()
1702
1703         self.assertEqual(stubs.capture_stdout.getvalue(),
1704                          stubs.expect_workflow_uuid + '\n')
1705         self.assertEqual(exited, 0)
1706
1707     @stubs()
1708     def test_create_with_no_input(self, stubs):
1709         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1710         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1711
1712         exited = arvados_cwl.main(
1713             ["--create-workflow", "--debug",
1714              "--api=containers",
1715              "--project-uuid", project_uuid,
1716              "tests/wf/revsort/revsort.cwl"],
1717             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1718
1719         stubs.api.pipeline_templates().create.refute_called()
1720         stubs.api.container_requests().create.refute_called()
1721
1722         self.assertEqual(stubs.capture_stdout.getvalue(),
1723                          stubs.expect_workflow_uuid + '\n')
1724         self.assertEqual(exited, 0)
1725
1726     @stubs()
1727     def test_create_map(self, stubs):
1728         # test uploading a document that uses objects instead of arrays
1729         # for certain fields like inputs and requirements.
1730
1731         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1732         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1733
1734         exited = arvados_cwl.main(
1735             ["--create-workflow", "--debug",
1736              "--api=containers",
1737              "--project-uuid", project_uuid,
1738              "--disable-git",
1739              "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
1740             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1741
1742         stubs.api.pipeline_templates().create.refute_called()
1743         stubs.api.container_requests().create.refute_called()
1744
1745         expect_workflow = StripYAMLComments(
1746             open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
1747
1748         body = {
1749             "workflow": {
1750                 "owner_uuid": project_uuid,
1751                 "name": "submit_wf_map.cwl",
1752                 "description": "",
1753                 "definition": expect_workflow,
1754             }
1755         }
1756         stubs.api.workflows().create.assert_called_with(
1757             body=JsonDiffMatcher(body))
1758
1759         self.assertEqual(stubs.capture_stdout.getvalue(),
1760                          stubs.expect_workflow_uuid + '\n')
1761         self.assertEqual(exited, 0)
1762
1763
1764 class TestPrintKeepDeps(unittest.TestCase):
1765     @stubs()
1766     def test_print_keep_deps(self, stubs):
1767         # test --print-keep-deps which is used by arv-copy
1768
1769         exited = arvados_cwl.main(
1770             ["--print-keep-deps", "--debug",
1771              "tests/wf/submit_wf_map.cwl"],
1772             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1773
1774         self.assertEqual(stubs.capture_stdout.getvalue(),
1775                          '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
1776         self.assertEqual(exited, 0)