sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfdetails=('submit_wf.cwl', None)):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62
  63             wfname = wfdetails[0]
  64             wfpath = wfdetails[1]
  65
  66             stubs = Stubs()
  67             stubs.events = events
  68             stubs.keepdocker = keepdocker
  69
  70             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  71                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  73                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6"]
  74
  75             determine_image_id.return_value = None
  76
  77             def putstub(p, **kwargs):
  78                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  79             keep_client1().put.side_effect = putstub
  80             keep_client1.put.side_effect = putstub
  81             keep_client2().put.side_effect = putstub
  82             keep_client2.put.side_effect = putstub
  83
  84             stubs.keep_client = keep_client2
  85             stubs.docker_images = {
  86                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  87                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  88                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  89                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  90             }
  91             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  92                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  93             stubs.keepdocker.side_effect = kd
  94
  95             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  96             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  97
  98             if sys.version_info[0] < 3:
  99                 stubs.capture_stdout = BytesIO()
 100             else:
 101                 stubs.capture_stdout = StringIO()
 102
 103             stubs.api = mock.MagicMock()
 104             stubs.api._rootDesc = get_rootDesc()
 105             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 106             stubs.api._rootDesc["revision"] = "20210628"
 107
 108             stubs.api.users().current().execute.return_value = {
 109                 "uuid": stubs.fake_user_uuid,
 110             }
 111             stubs.api.collections().list().execute.return_value = {"items": []}
 112             stubs.api.containers().current().execute.return_value = {
 113                 "uuid": stubs.fake_container_uuid,
 114             }
 115             stubs.api.config()["StorageClasses"].items.return_value = {
 116                 "default": {
 117                     "Default": True
 118                 }
 119             }.items()
 120
 121             class CollectionExecute(object):
 122                 def __init__(self, exe):
 123                     self.exe = exe
 124                 def execute(self, num_retries=None):
 125                     return self.exe
 126
 127             def collection_createstub(created_collections, body, ensure_unique_name=None):
 128                 mt = body["manifest_text"].encode('utf-8')
 129                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 130                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 131                 created_collections[uuid] = {
 132                     "uuid": uuid,
 133                     "portable_data_hash": pdh,
 134                     "manifest_text": mt.decode('utf-8')
 135                 }
 136                 return CollectionExecute(created_collections[uuid])
 137
 138             def collection_getstub(created_collections, uuid):
 139                 for v in viewvalues(created_collections):
 140                     if uuid in (v["uuid"], v["portable_data_hash"]):
 141                         return CollectionExecute(v)
 142
 143             created_collections = {
 144                 "99999999999999999999999999999998+99": {
 145                     "uuid": "",
 146                     "portable_data_hash": "99999999999999999999999999999998+99",
 147                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 148                 },
 149                 "99999999999999999999999999999997+99": {
 150                     "uuid": "",
 151                     "portable_data_hash": "99999999999999999999999999999997+99",
 152                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 153                 },
 154                 "99999999999999999999999999999994+99": {
 155                     "uuid": "",
 156                     "portable_data_hash": "99999999999999999999999999999994+99",
 157                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 158                 },
 159                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 160                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 161                     "portable_data_hash": "999999999999999999999999999999d3+99",
 162                     "manifest_text": ""
 163                 },
 164                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 165                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 166                     "portable_data_hash": "999999999999999999999999999999d4+99",
 167                     "manifest_text": ""
 168                 },
 169                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 170                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 171                     "portable_data_hash": "999999999999999999999999999999d5+99",
 172                     "manifest_text": ""
 173                 },
 174                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 175                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 176                     "portable_data_hash": "999999999999999999999999999999d6+99",
 177                     "manifest_text": ""
 178                 }
 179             }
 180             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 181             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 182
 183             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 184             stubs.api.container_requests().create().execute.return_value = {
 185                 "uuid": stubs.expect_container_request_uuid,
 186                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 187                 "state": "Queued"
 188             }
 189
 190             cwd = os.getcwd()
 191             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 192             with open(filepath) as f:
 193                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 194                 expect_packed_workflow = yaml.load(f)
 195
 196             if wfpath is None:
 197                 wfpath = wfname
 198
 199             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 200             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 201             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 202
 203             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 204             expect_packed_workflow.update(stubs.git_info)
 205
 206             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 207
 208             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 209             if wfname == wfpath:
 210                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 211             else:
 212                 container_name = wfname
 213
 214             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 215             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 216
 217             stubs.expect_container_spec = {
 218                 'priority': 500,
 219                 'mounts': {
 220                     '/var/spool/cwl': {
 221                         'writable': True,
 222                         'kind': 'collection'
 223                     },
 224                     '/var/lib/cwl/workflow.json': {
 225                         'content': expect_packed_workflow,
 226                         'kind': 'json'
 227                     },
 228                     'stdout': {
 229                         'path': '/var/spool/cwl/cwl.output.json',
 230                         'kind': 'file'
 231                     },
 232                     '/var/lib/cwl/cwl.input.json': {
 233                         'kind': 'json',
 234                         'content': {
 235                             'y': {
 236                                 'basename': '99999999999999999999999999999998+99',
 237                                 'location': 'keep:99999999999999999999999999999998+99',
 238                                 'class': 'Directory'},
 239                             'x': {
 240                                 'basename': u'blorp.txt',
 241                                 'class': 'File',
 242                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 243                                 "size": 16
 244                             },
 245                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 246                                 {'basename': 'renamed.txt',
 247                                  'class': 'File',
 248                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 249                                  'size': 0
 250                                 }
 251                             ]}
 252                         },
 253                         'kind': 'json'
 254                     }
 255                 },
 256                 'secret_mounts': {},
 257                 'state': 'Committed',
 258                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 259                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 260                             '--eval-timeout=20', '--thread-count=0',
 261                             '--enable-reuse', "--collection-cache-size=256",
 262                             '--output-name=Output from workflow '+container_name,
 263                             '--debug', '--on-error=continue',
 264                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 265                 'name': container_name,
 266                 'container_image': '999999999999999999999999999999d3+99',
 267                 'output_name': 'Output from workflow %s' % (container_name),
 268                 'output_path': '/var/spool/cwl',
 269                 'cwd': '/var/spool/cwl',
 270                 'runtime_constraints': {
 271                     'API': True,
 272                     'vcpus': 1,
 273                     'ram': (1024+256)*1024*1024
 274                 },
 275                 'properties': stubs.git_props,
 276                 'use_existing': False,
 277                 'secret_mounts': {}
 278             }
 279
 280             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 281             stubs.api.workflows().create().execute.return_value = {
 282                 "uuid": stubs.expect_workflow_uuid,
 283             }
 284             def update_mock(**kwargs):
 285                 stubs.updated_uuid = kwargs.get('uuid')
 286                 return mock.DEFAULT
 287             stubs.api.workflows().update.side_effect = update_mock
 288             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 289                 "uuid": stubs.updated_uuid,
 290             }
 291
 292             return func(self, stubs, *args, **kwargs)
 293         return wrapped
 294     return outer_wrapper
 295
 296 class TestSubmit(unittest.TestCase):
 297
 298     def setUp(self):
 299         cwltool.process._names = set()
 300         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 301
 302     def tearDown(self):
 303         root_logger = logging.getLogger('')
 304
 305         # Remove existing RuntimeStatusLoggingHandlers if they exist
 306         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 307         root_logger.handlers = handlers
 308
 309     @mock.patch("time.sleep")
 310     @stubs()
 311     def test_submit_invalid_runner_ram(self, stubs, tm):
 312         exited = arvados_cwl.main(
 313             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 316         self.assertEqual(exited, 1)
 317
 318
 319     @stubs()
 320     def test_submit_container(self, stubs):
 321         exited = arvados_cwl.main(
 322             ["--submit", "--no-wait", "--api=containers", "--debug",
 323                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 324             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 325
 326         stubs.api.collections().create.assert_has_calls([
 327             mock.call(body=JsonDiffMatcher({
 328                 'manifest_text':
 329                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 330                 'replication_desired': None,
 331                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 332             }), ensure_unique_name=False),
 333             mock.call(body=JsonDiffMatcher({
 334                 'manifest_text':
 335                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 336                 'replication_desired': None,
 337                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 338             }), ensure_unique_name=False),
 339             ])
 340
 341         expect_container = copy.deepcopy(stubs.expect_container_spec)
 342         stubs.api.container_requests().create.assert_called_with(
 343             body=JsonDiffMatcher(expect_container))
 344         self.assertEqual(stubs.capture_stdout.getvalue(),
 345                          stubs.expect_container_request_uuid + '\n')
 346         self.assertEqual(exited, 0)
 347
 348
 349     @stubs()
 350     def test_submit_container_tool(self, stubs):
 351         # test for issue #16139
 352         exited = arvados_cwl.main(
 353             ["--submit", "--no-wait", "--api=containers", "--debug",
 354                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 355             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 356
 357         self.assertEqual(stubs.capture_stdout.getvalue(),
 358                          stubs.expect_container_request_uuid + '\n')
 359         self.assertEqual(exited, 0)
 360
 361     @stubs()
 362     def test_submit_container_no_reuse(self, stubs):
 363         exited = arvados_cwl.main(
 364             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 365                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 366             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 367
 368         expect_container = copy.deepcopy(stubs.expect_container_spec)
 369         expect_container["command"] = [
 370             'arvados-cwl-runner', '--local', '--api=containers',
 371             '--no-log-timestamps', '--disable-validate', '--disable-color',
 372             '--eval-timeout=20', '--thread-count=0',
 373             '--disable-reuse', "--collection-cache-size=256",
 374             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 375             '--debug', '--on-error=continue',
 376             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 377         expect_container["use_existing"] = False
 378
 379         stubs.api.container_requests().create.assert_called_with(
 380             body=JsonDiffMatcher(expect_container))
 381         self.assertEqual(stubs.capture_stdout.getvalue(),
 382                          stubs.expect_container_request_uuid + '\n')
 383         self.assertEqual(exited, 0)
 384
 385     @stubs(('submit_wf_no_reuse.cwl', None))
 386     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 387         exited = arvados_cwl.main(
 388             ["--submit", "--no-wait", "--api=containers", "--debug",
 389              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 390             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 391         self.assertEqual(exited, 0)
 392
 393         expect_container = copy.deepcopy(stubs.expect_container_spec)
 394         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 395         expect_container["use_existing"] = False
 396         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 397             {
 398                 "class": "WorkReuse",
 399                 "enableReuse": False,
 400             },
 401                 {
 402                     "acrContainerImage": "999999999999999999999999999999d3+99",
 403                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 404                 }
 405         ]
 406         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl"
 407
 408         stubs.api.container_requests().create.assert_called_with(
 409             body=JsonDiffMatcher(expect_container))
 410         self.assertEqual(stubs.capture_stdout.getvalue(),
 411                          stubs.expect_container_request_uuid + '\n')
 412
 413
 414     @stubs()
 415     def test_submit_container_on_error(self, stubs):
 416         exited = arvados_cwl.main(
 417             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 418                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 419             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 420
 421         expect_container = copy.deepcopy(stubs.expect_container_spec)
 422         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 423                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 424                                        '--eval-timeout=20', '--thread-count=0',
 425                                        '--enable-reuse', "--collection-cache-size=256",
 426                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 427                                        '--debug', '--on-error=stop',
 428                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 429
 430         stubs.api.container_requests().create.assert_called_with(
 431             body=JsonDiffMatcher(expect_container))
 432         self.assertEqual(stubs.capture_stdout.getvalue(),
 433                          stubs.expect_container_request_uuid + '\n')
 434         self.assertEqual(exited, 0)
 435
 436     @stubs()
 437     def test_submit_container_output_name(self, stubs):
 438         output_name = "test_output_name"
 439
 440         exited = arvados_cwl.main(
 441             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 442                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 443             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 444
 445         expect_container = copy.deepcopy(stubs.expect_container_spec)
 446         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 447                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 448                                        '--eval-timeout=20', '--thread-count=0',
 449                                        '--enable-reuse', "--collection-cache-size=256",
 450                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 451                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 452         expect_container["output_name"] = output_name
 453
 454         stubs.api.container_requests().create.assert_called_with(
 455             body=JsonDiffMatcher(expect_container))
 456         self.assertEqual(stubs.capture_stdout.getvalue(),
 457                          stubs.expect_container_request_uuid + '\n')
 458         self.assertEqual(exited, 0)
 459
 460     @stubs()
 461     def test_submit_storage_classes(self, stubs):
 462         exited = arvados_cwl.main(
 463             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 464                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 465             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 466
 467         expect_container = copy.deepcopy(stubs.expect_container_spec)
 468         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 469                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 470                                        '--eval-timeout=20', '--thread-count=0',
 471                                        '--enable-reuse', "--collection-cache-size=256",
 472                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 473                                        "--debug",
 474                                        "--storage-classes=foo", '--on-error=continue',
 475                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 476
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(stubs.capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481         self.assertEqual(exited, 0)
 482
 483     @stubs()
 484     def test_submit_multiple_storage_classes(self, stubs):
 485         exited = arvados_cwl.main(
 486             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 487                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 488             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 489
 490         expect_container = copy.deepcopy(stubs.expect_container_spec)
 491         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 492                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 493                                        '--eval-timeout=20', '--thread-count=0',
 494                                        '--enable-reuse', "--collection-cache-size=256",
 495                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 496                                        "--debug",
 497                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 498                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 499
 500         stubs.api.container_requests().create.assert_called_with(
 501             body=JsonDiffMatcher(expect_container))
 502         self.assertEqual(stubs.capture_stdout.getvalue(),
 503                          stubs.expect_container_request_uuid + '\n')
 504         self.assertEqual(exited, 0)
 505
 506     @mock.patch("cwltool.task_queue.TaskQueue")
 507     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 508     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 509     @stubs()
 510     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 511         final_output_c = arvados.collection.Collection()
 512         make_output.return_value = ({},final_output_c)
 513
 514         def set_final_output(job_order, output_callback, runtimeContext):
 515             output_callback({"out": "zzzzz"}, "success")
 516             return []
 517         job.side_effect = set_final_output
 518
 519         exited = arvados_cwl.main(
 520             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 521                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 522             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 523
 524         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 525         self.assertEqual(exited, 0)
 526
 527     @mock.patch("cwltool.task_queue.TaskQueue")
 528     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 529     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 530     @stubs()
 531     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 532         final_output_c = arvados.collection.Collection()
 533         make_output.return_value = ({},final_output_c)
 534         stubs.api.config().get.return_value = {"default": {"Default": True}}
 535
 536         def set_final_output(job_order, output_callback, runtimeContext):
 537             output_callback({"out": "zzzzz"}, "success")
 538             return []
 539         job.side_effect = set_final_output
 540
 541         exited = arvados_cwl.main(
 542             ["--debug", "--local", "--disable-git",
 543                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 544             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 545
 546         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 547         self.assertEqual(exited, 0)
 548
 549     @mock.patch("cwltool.task_queue.TaskQueue")
 550     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 551     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 552     @stubs()
 553     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 554         final_output_c = arvados.collection.Collection()
 555         make_output.return_value = ({},final_output_c)
 556
 557         def set_final_output(job_order, output_callback, runtimeContext):
 558             output_callback({"out": "zzzzz"}, "success")
 559             return []
 560         job.side_effect = set_final_output
 561
 562         exited = arvados_cwl.main(
 563             ["--debug", "--local", "--disable-git",
 564                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 565             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 566
 567         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 568         self.assertEqual(exited, 0)
 569
 570     @stubs()
 571     def test_submit_container_output_ttl(self, stubs):
 572         exited = arvados_cwl.main(
 573             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 574                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 575             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 576
 577         expect_container = copy.deepcopy(stubs.expect_container_spec)
 578         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 579                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 580                                        '--eval-timeout=20', '--thread-count=0',
 581                                        '--enable-reuse', "--collection-cache-size=256",
 582                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 583                                        '--debug',
 584                                        '--on-error=continue',
 585                                        "--intermediate-output-ttl=3600",
 586                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 587
 588         stubs.api.container_requests().create.assert_called_with(
 589             body=JsonDiffMatcher(expect_container))
 590         self.assertEqual(stubs.capture_stdout.getvalue(),
 591                          stubs.expect_container_request_uuid + '\n')
 592         self.assertEqual(exited, 0)
 593
 594     @stubs()
 595     def test_submit_container_trash_intermediate(self, stubs):
 596         exited = arvados_cwl.main(
 597             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 598                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 599             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 600
 601
 602         expect_container = copy.deepcopy(stubs.expect_container_spec)
 603         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 604                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 605                                        '--eval-timeout=20', '--thread-count=0',
 606                                        '--enable-reuse', "--collection-cache-size=256",
 607                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 608                                        '--debug', '--on-error=continue',
 609                                        "--trash-intermediate",
 610                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 611
 612         stubs.api.container_requests().create.assert_called_with(
 613             body=JsonDiffMatcher(expect_container))
 614         self.assertEqual(stubs.capture_stdout.getvalue(),
 615                          stubs.expect_container_request_uuid + '\n')
 616         self.assertEqual(exited, 0)
 617
 618     @stubs()
 619     def test_submit_container_output_tags(self, stubs):
 620         output_tags = "tag0,tag1,tag2"
 621
 622         exited = arvados_cwl.main(
 623             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 624                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 625             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 626
 627         expect_container = copy.deepcopy(stubs.expect_container_spec)
 628         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 629                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 630                                        '--eval-timeout=20', '--thread-count=0',
 631                                        '--enable-reuse', "--collection-cache-size=256",
 632                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 633                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 634                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 635
 636         stubs.api.container_requests().create.assert_called_with(
 637             body=JsonDiffMatcher(expect_container))
 638         self.assertEqual(stubs.capture_stdout.getvalue(),
 639                          stubs.expect_container_request_uuid + '\n')
 640         self.assertEqual(exited, 0)
 641
 642     @stubs()
 643     def test_submit_container_runner_ram(self, stubs):
 644         exited = arvados_cwl.main(
 645             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 646                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 647             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 648
 649         expect_container = copy.deepcopy(stubs.expect_container_spec)
 650         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 651         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 652                 {
 653                     "acrContainerImage": "999999999999999999999999999999d3+99",
 654                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 655                     "ramMin": 2048
 656                 }
 657         ]
 658
 659         stubs.api.container_requests().create.assert_called_with(
 660             body=JsonDiffMatcher(expect_container))
 661         self.assertEqual(stubs.capture_stdout.getvalue(),
 662                          stubs.expect_container_request_uuid + '\n')
 663         self.assertEqual(exited, 0)
 664
 665     @mock.patch("arvados.collection.CollectionReader")
 666     @mock.patch("time.sleep")
 667     @stubs()
 668     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 669         collectionReader().exists.return_value = True
 670         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 671         exited = arvados_cwl.main(
 672             ["--submit", "--no-wait", "--api=containers", "--debug",
 673              "tests/wf/submit_keepref_wf.cwl"],
 674             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 675         self.assertEqual(exited, 0)
 676
 677     @mock.patch("arvados.collection.CollectionReader")
 678     @mock.patch("time.sleep")
 679     @stubs()
 680     def test_submit_keepref(self, stubs, tm, reader):
 681         raise Exception("broken")
 682
 683         with open("tests/wf/expect_arvworkflow.cwl") as f:
 684             reader().open().__enter__().read.return_value = f.read()
 685
 686         exited = arvados_cwl.main(
 687             ["--submit", "--no-wait", "--api=containers", "--debug",
 688              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 689             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 690
 691         expect_container = {
 692             'priority': 500,
 693             'mounts': {
 694                 '/var/spool/cwl': {
 695                     'writable': True,
 696                     'kind': 'collection'
 697                 },
 698                 'stdout': {
 699                     'path': '/var/spool/cwl/cwl.output.json',
 700                     'kind': 'file'
 701                 },
 702                 '/var/lib/cwl/workflow': {
 703                     'portable_data_hash': '99999999999999999999999999999994+99',
 704                     'kind': 'collection'
 705                 },
 706                 '/var/lib/cwl/cwl.input.json': {
 707                     'content': {
 708                         'x': 'XxX'
 709                     },
 710                     'kind': 'json'
 711                 }
 712             }, 'state': 'Committed',
 713             'output_path': '/var/spool/cwl',
 714             'name': 'expect_arvworkflow.cwl#main',
 715             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 716             'container_image': '999999999999999999999999999999d3+99',
 717             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 718                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 719                         '--eval-timeout=20', '--thread-count=0',
 720                         '--enable-reuse', "--collection-cache-size=256",
 721                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 722                         '--debug', '--on-error=continue',
 723                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 724             'cwd': '/var/spool/cwl',
 725             'runtime_constraints': {
 726                 'API': True,
 727                 'vcpus': 1,
 728                 'ram': 1342177280
 729             },
 730             'use_existing': False,
 731             'properties': {},
 732             'secret_mounts': {}
 733         }
 734
 735         stubs.api.container_requests().create.assert_called_with(
 736             body=JsonDiffMatcher(expect_container))
 737         self.assertEqual(stubs.capture_stdout.getvalue(),
 738                          stubs.expect_container_request_uuid + '\n')
 739         self.assertEqual(exited, 0)
 740
 741     @mock.patch("time.sleep")
 742     @stubs()
 743     def test_submit_arvworkflow(self, stubs, tm):
 744         with open("tests/wf/expect_arvworkflow.cwl") as f:
 745             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 746
 747         exited = arvados_cwl.main(
 748             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 749              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 750             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 751
 752         expect_container = {
 753             'priority': 500,
 754             'mounts': {
 755                 '/var/spool/cwl': {
 756                     'writable': True,
 757                     'kind': 'collection'
 758                 },
 759                 'stdout': {
 760                     'path': '/var/spool/cwl/cwl.output.json',
 761                     'kind': 'file'
 762                 },
 763                 '/var/lib/cwl/workflow.json': {
 764                     'kind': 'json',
 765                     'content': {
 766                         'cwlVersion': 'v1.0',
 767                         'label': 'a test workflow',
 768                         '$graph': [
 769                             {
 770                                 'id': '#main',
 771                                 'inputs': [
 772                                     {'type': 'string', 'id': '#main/x'}
 773                                 ],
 774                                 'steps': [
 775                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 776                                      'run': '#submit_tool.cwl',
 777                                      'id': '#main/step1',
 778                                      'out': []}
 779                                 ],
 780                                 'class': 'Workflow',
 781                                 'outputs': []
 782                             },
 783                             {
 784                                 'inputs': [
 785                                     {
 786                                         'inputBinding': {'position': 1},
 787                                         'type': 'string',
 788                                         'id': '#submit_tool.cwl/x'}
 789                                 ],
 790                                 'requirements': [
 791                                     {
 792                                         'dockerPull': 'debian:buster-slim',
 793                                         'class': 'DockerRequirement'
 794                                     }
 795                                 ],
 796                                 'id': '#submit_tool.cwl',
 797                                 'outputs': [],
 798                                 'baseCommand': 'cat',
 799                                 'class': 'CommandLineTool'
 800                             }
 801                         ]
 802                     }
 803                 },
 804                 '/var/lib/cwl/cwl.input.json': {
 805                     'content': {
 806                         'x': 'XxX'
 807                     },
 808                     'kind': 'json'
 809                 }
 810             }, 'state': 'Committed',
 811             'output_path': '/var/spool/cwl',
 812             'name': 'a test workflow',
 813             'container_image': "999999999999999999999999999999d3+99",
 814             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 815                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 816                         '--eval-timeout=20', '--thread-count=0',
 817                         '--enable-reuse', "--collection-cache-size=256",
 818                         "--output-name=Output from workflow a test workflow",
 819                         '--debug', '--on-error=continue',
 820                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 821             'output_name': 'Output from workflow a test workflow',
 822             'cwd': '/var/spool/cwl',
 823             'runtime_constraints': {
 824                 'API': True,
 825                 'vcpus': 1,
 826                 'ram': 1342177280
 827             },
 828             'use_existing': False,
 829             'properties': {
 830                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 831             },
 832             'secret_mounts': {}
 833         }
 834
 835         stubs.api.container_requests().create.assert_called_with(
 836             body=JsonDiffMatcher(expect_container))
 837         self.assertEqual(stubs.capture_stdout.getvalue(),
 838                          stubs.expect_container_request_uuid + '\n')
 839         self.assertEqual(exited, 0)
 840
 841     @stubs(('hello container 123', 'submit_wf.cwl'))
 842     def test_submit_container_name(self, stubs):
 843         exited = arvados_cwl.main(
 844             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 845                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 846             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 847
 848         expect_container = copy.deepcopy(stubs.expect_container_spec)
 849
 850         stubs.api.container_requests().create.assert_called_with(
 851             body=JsonDiffMatcher(expect_container))
 852         self.assertEqual(stubs.capture_stdout.getvalue(),
 853                          stubs.expect_container_request_uuid + '\n')
 854         self.assertEqual(exited, 0)
 855
 856     @stubs()
 857     def test_submit_missing_input(self, stubs):
 858         exited = arvados_cwl.main(
 859             ["--submit", "--no-wait", "--api=containers", "--debug",
 860              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 861             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 862         self.assertEqual(exited, 0)
 863
 864         exited = arvados_cwl.main(
 865             ["--submit", "--no-wait", "--api=containers", "--debug",
 866              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 867             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 868         self.assertEqual(exited, 1)
 869
 870     @stubs()
 871     def test_submit_container_project(self, stubs):
 872         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 873         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 874         exited = arvados_cwl.main(
 875             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 876                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 877             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 878
 879         expect_container = copy.deepcopy(stubs.expect_container_spec)
 880         expect_container["owner_uuid"] = project_uuid
 881         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 882                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 883                                        "--eval-timeout=20", "--thread-count=0",
 884                                        '--enable-reuse', "--collection-cache-size=256",
 885                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 886                                        '--debug',
 887                                        '--on-error=continue',
 888                                        '--project-uuid='+project_uuid,
 889                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 890
 891         stubs.api.container_requests().create.assert_called_with(
 892             body=JsonDiffMatcher(expect_container))
 893         self.assertEqual(stubs.capture_stdout.getvalue(),
 894                          stubs.expect_container_request_uuid + '\n')
 895         self.assertEqual(exited, 0)
 896
 897     @stubs()
 898     def test_submit_container_eval_timeout(self, stubs):
 899         exited = arvados_cwl.main(
 900             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 901                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 902             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 903
 904         expect_container = copy.deepcopy(stubs.expect_container_spec)
 905         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 906                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 907                                        '--eval-timeout=60.0', '--thread-count=0',
 908                                        '--enable-reuse', "--collection-cache-size=256",
 909                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 910                                        '--debug', '--on-error=continue',
 911                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 912
 913         stubs.api.container_requests().create.assert_called_with(
 914             body=JsonDiffMatcher(expect_container))
 915         self.assertEqual(stubs.capture_stdout.getvalue(),
 916                          stubs.expect_container_request_uuid + '\n')
 917         self.assertEqual(exited, 0)
 918
 919     @stubs()
 920     def test_submit_container_collection_cache(self, stubs):
 921         exited = arvados_cwl.main(
 922             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 923                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 924             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 925
 926         expect_container = copy.deepcopy(stubs.expect_container_spec)
 927         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 928                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 929                                        '--eval-timeout=20', '--thread-count=0',
 930                                        '--enable-reuse', "--collection-cache-size=500",
 931                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 932                                        '--debug', '--on-error=continue',
 933                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 934         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 935
 936         stubs.api.container_requests().create.assert_called_with(
 937             body=JsonDiffMatcher(expect_container))
 938         self.assertEqual(stubs.capture_stdout.getvalue(),
 939                          stubs.expect_container_request_uuid + '\n')
 940         self.assertEqual(exited, 0)
 941
 942     @stubs()
 943     def test_submit_container_thread_count(self, stubs):
 944         exited = arvados_cwl.main(
 945             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 946                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 947             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 948
 949         expect_container = copy.deepcopy(stubs.expect_container_spec)
 950         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 951                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 952                                        '--eval-timeout=20', '--thread-count=20',
 953                                        '--enable-reuse', "--collection-cache-size=256",
 954                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 955                                        '--debug', '--on-error=continue',
 956                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 957
 958         stubs.api.container_requests().create.assert_called_with(
 959             body=JsonDiffMatcher(expect_container))
 960         self.assertEqual(stubs.capture_stdout.getvalue(),
 961                          stubs.expect_container_request_uuid + '\n')
 962         self.assertEqual(exited, 0)
 963
 964     @stubs()
 965     def test_submit_container_runner_image(self, stubs):
 966         exited = arvados_cwl.main(
 967             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 968                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 969             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 970
 971         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 972         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 973                 {
 974                     "acrContainerImage": "999999999999999999999999999999d5+99",
 975                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 976                 }
 977         ]
 978
 979         expect_container = copy.deepcopy(stubs.expect_container_spec)
 980         stubs.api.container_requests().create.assert_called_with(
 981             body=JsonDiffMatcher(expect_container))
 982         self.assertEqual(stubs.capture_stdout.getvalue(),
 983                          stubs.expect_container_request_uuid + '\n')
 984         self.assertEqual(exited, 0)
 985
 986     @stubs()
 987     def test_submit_priority(self, stubs):
 988         exited = arvados_cwl.main(
 989             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 990                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 991             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 992
 993         stubs.expect_container_spec["priority"] = 669
 994
 995         expect_container = copy.deepcopy(stubs.expect_container_spec)
 996         stubs.api.container_requests().create.assert_called_with(
 997             body=JsonDiffMatcher(expect_container))
 998         self.assertEqual(stubs.capture_stdout.getvalue(),
 999                          stubs.expect_container_request_uuid + '\n')
1000         self.assertEqual(exited, 0)
1001
1002     @stubs(('submit_wf_runner_resources.cwl', None))
1003     def test_submit_wf_runner_resources(self, stubs):
1004         exited = arvados_cwl.main(
1005             ["--submit", "--no-wait", "--api=containers", "--debug",
1006                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1007             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1008
1009         expect_container = copy.deepcopy(stubs.expect_container_spec)
1010         expect_container["runtime_constraints"] = {
1011             "API": True,
1012             "vcpus": 2,
1013             "ram": (2000+512) * 2**20
1014         }
1015         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1016             {
1017                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1018                 "acrContainerImage": "999999999999999999999999999999d3+99",
1019                 "coresMin": 2,
1020                 "ramMin": 2000,
1021                 "keep_cache": 512
1022             }
1023         ]
1024         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1025         #    "arv": "http://arvados.org/cwl#",
1026         #}
1027         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1028         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl"
1029
1030         stubs.api.container_requests().create.assert_called_with(
1031             body=JsonDiffMatcher(expect_container))
1032         self.assertEqual(stubs.capture_stdout.getvalue(),
1033                          stubs.expect_container_request_uuid + '\n')
1034         self.assertEqual(exited, 0)
1035
1036     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1037     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1038     @mock.patch("arvados.api")
1039     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1040         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1041
1042         arvrunner = mock.MagicMock()
1043         arvrunner.project_uuid = ""
1044         api.return_value = mock.MagicMock()
1045         arvrunner.api = api.return_value
1046         arvrunner.runtimeContext.match_local_docker = False
1047         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1048                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1049                                                                         "link_class": "docker_image_repo+tag",
1050                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1051                                                                         "owner_uuid": "",
1052                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1053                                                             {"items": [{"created_at": "",
1054                                                                         "head_uuid": "",
1055                                                                         "link_class": "docker_image_hash",
1056                                                                         "name": "123456",
1057                                                                         "owner_uuid": "",
1058                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1059                                                             {"items": [{"created_at": "",
1060                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1061                                                                         "link_class": "docker_image_repo+tag",
1062                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1063                                                                         "owner_uuid": "",
1064                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1065                                                             {"items": [{"created_at": "",
1066                                                                         "head_uuid": "",
1067                                                                         "link_class": "docker_image_hash",
1068                                                                         "name": "123456",
1069                                                                         "owner_uuid": "",
1070                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1071         )
1072         find_one_image_hash.return_value = "123456"
1073
1074         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1075                                                                               "owner_uuid": "",
1076                                                                               "manifest_text": "",
1077                                                                               "properties": ""
1078                                                                               }], "items_available": 1, "offset": 0},
1079                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1080                                                                               "owner_uuid": "",
1081                                                                               "manifest_text": "",
1082                                                                               "properties": ""
1083                                                                           }], "items_available": 1, "offset": 0})
1084         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1085         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1086                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1087
1088         self.assertEqual("9999999999999999999999999999999b+99",
1089                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1090
1091
1092     @stubs()
1093     def test_submit_secrets(self, stubs):
1094         exited = arvados_cwl.main(
1095             ["--submit", "--no-wait", "--api=containers", "--debug",
1096                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1097             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1098
1099         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1100         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1101
1102         expect_container = {
1103             "command": [
1104                 "arvados-cwl-runner",
1105                 "--local",
1106                 "--api=containers",
1107                 "--no-log-timestamps",
1108                 "--disable-validate",
1109                 "--disable-color",
1110                 "--eval-timeout=20",
1111                 '--thread-count=0',
1112                 "--enable-reuse",
1113                 "--collection-cache-size=256",
1114                 '--output-name=Output from workflow secret_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1115                 "--debug",
1116                 "--on-error=continue",
1117                 "/var/lib/cwl/workflow.json#main",
1118                 "/var/lib/cwl/cwl.input.json"
1119             ],
1120             "container_image": "999999999999999999999999999999d3+99",
1121             "cwd": "/var/spool/cwl",
1122             "mounts": {
1123                 "/var/lib/cwl/cwl.input.json": {
1124                     "content": {
1125                         "pw": {
1126                             "$include": "/secrets/s0"
1127                         }
1128                     },
1129                     "kind": "json"
1130                 },
1131                 "/var/lib/cwl/workflow.json": {
1132                     "content": {
1133                         "$graph": [
1134                             {
1135                                 "class": "Workflow",
1136                                 "hints": [
1137                                     {
1138                                     "class": "DockerRequirement",
1139                                     "dockerPull": "debian:buster-slim",
1140                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1141                                     },
1142                                     {
1143                                         "class": "http://commonwl.org/cwltool#Secrets",
1144                                         "secrets": [
1145                                             "#main/pw"
1146                                         ]
1147                                     },
1148                                     {
1149                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1150                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1151                                     }
1152                                 ],
1153                                 "id": "#main",
1154                                 "inputs": [
1155                                     {
1156                                         "id": "#main/pw",
1157                                         "type": "string"
1158                                     }
1159                                 ],
1160                                 "outputs": [
1161                                     {
1162                                         "id": "#main/out",
1163                                         "outputSource": "#main/step/out",
1164                                         "type": "File"
1165                                     }
1166                                 ],
1167                                 "requirements": [
1168                                     {
1169                                         "class": "SubworkflowFeatureRequirement"
1170                                     }
1171                                 ],
1172                                 "steps": [
1173                                     {
1174                                         "id": "#main/secret_wf.cwl (2.5.0-15-g165ec76d4)",
1175                                         "in": [
1176                                             {
1177                                                 "id": "#main/step/pw",
1178                                                 "source": "#main/pw"
1179                                             }
1180                                         ],
1181                                         "label": "secret_wf.cwl (2.5.0-15-g165ec76d4)",
1182                                         "out": [
1183                                             {"id": "#main/step/out"}
1184                                         ],
1185                                         "run": "keep:702edbfca88e907b62234556089dbce7+119/secret_wf.cwl"
1186                                     }
1187                                 ]
1188                             }
1189                         ],
1190                         "cwlVersion": "v1.2"
1191                     },
1192                     "kind": "json"
1193                 },
1194                 "/var/spool/cwl": {
1195                     "kind": "collection",
1196                     "writable": True
1197                 },
1198                 "stdout": {
1199                     "kind": "file",
1200                     "path": "/var/spool/cwl/cwl.output.json"
1201                 }
1202             },
1203             "name": "secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1204             "output_name": "Output from workflow secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1205             "output_path": "/var/spool/cwl",
1206             "priority": 500,
1207             "properties": stubs.git_props,
1208             "runtime_constraints": {
1209                 "API": True,
1210                 "ram": 1342177280,
1211                 "vcpus": 1
1212             },
1213             "secret_mounts": {
1214                 "/secrets/s0": {
1215                     "content": "blorp",
1216                     "kind": "text"
1217                 }
1218             },
1219             "state": "Committed",
1220             "use_existing": False
1221         }
1222
1223         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"].update(stubs.git_info)
1224
1225         stubs.api.container_requests().create.assert_called_with(
1226             body=JsonDiffMatcher(expect_container))
1227         self.assertEqual(stubs.capture_stdout.getvalue(),
1228                          stubs.expect_container_request_uuid + '\n')
1229         self.assertEqual(exited, 0)
1230
1231     @stubs()
1232     def test_submit_request_uuid(self, stubs):
1233         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1234         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1235
1236         stubs.api.container_requests().update().execute.return_value = {
1237             "uuid": stubs.expect_container_request_uuid,
1238             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1239             "state": "Queued"
1240         }
1241
1242         exited = arvados_cwl.main(
1243             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1244                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1245             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1246
1247         stubs.api.container_requests().update.assert_called_with(
1248             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1249         self.assertEqual(stubs.capture_stdout.getvalue(),
1250                          stubs.expect_container_request_uuid + '\n')
1251         self.assertEqual(exited, 0)
1252
1253     @stubs()
1254     def test_submit_container_cluster_id(self, stubs):
1255         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1256
1257         exited = arvados_cwl.main(
1258             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1259                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1260             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1261
1262         expect_container = copy.deepcopy(stubs.expect_container_spec)
1263
1264         stubs.api.container_requests().create.assert_called_with(
1265             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1266         self.assertEqual(stubs.capture_stdout.getvalue(),
1267                          stubs.expect_container_request_uuid + '\n')
1268         self.assertEqual(exited, 0)
1269
1270     @stubs()
1271     def test_submit_validate_cluster_id(self, stubs):
1272         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1273         exited = arvados_cwl.main(
1274             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1275              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1276             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1277         self.assertEqual(exited, 1)
1278
1279     @stubs()
1280     def test_submit_validate_project_uuid(self, stubs):
1281         # Fails with bad cluster prefix
1282         exited = arvados_cwl.main(
1283             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1284              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1285             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1286         self.assertEqual(exited, 1)
1287
1288         # Project lookup fails
1289         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1290         exited = arvados_cwl.main(
1291             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1292              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1293             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1294         self.assertEqual(exited, 1)
1295
1296         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1297         exited = arvados_cwl.main(
1298             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1299              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1300             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1301         self.assertEqual(exited, 0)
1302
1303
1304     @mock.patch("arvados.collection.CollectionReader")
1305     @stubs()
1306     def test_submit_uuid_inputs(self, stubs, collectionReader):
1307         collectionReader().exists.return_value = True
1308         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1309         def list_side_effect(**kwargs):
1310             m = mock.MagicMock()
1311             if "count" in kwargs:
1312                 m.execute.return_value = {"items": [
1313                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1314                 ]}
1315             else:
1316                 m.execute.return_value = {"items": []}
1317             return m
1318         stubs.api.collections().list.side_effect = list_side_effect
1319         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1320
1321         exited = arvados_cwl.main(
1322             ["--submit", "--no-wait", "--api=containers", "--debug",
1323                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1324             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1325
1326         expect_container = copy.deepcopy(stubs.expect_container_spec)
1327         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1328         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1329         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1330
1331         stubs.api.collections().list.assert_has_calls([
1332             mock.call(count='none',
1333                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1334                       select=['uuid', 'portable_data_hash'])])
1335         stubs.api.container_requests().create.assert_called_with(
1336             body=JsonDiffMatcher(expect_container))
1337         self.assertEqual(stubs.capture_stdout.getvalue(),
1338                          stubs.expect_container_request_uuid + '\n')
1339         self.assertEqual(exited, 0)
1340
1341     @stubs()
1342     def test_submit_mismatched_uuid_inputs(self, stubs):
1343         def list_side_effect(**kwargs):
1344             m = mock.MagicMock()
1345             if "count" in kwargs:
1346                 m.execute.return_value = {"items": [
1347                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1348                 ]}
1349             else:
1350                 m.execute.return_value = {"items": []}
1351             return m
1352         stubs.api.collections().list.side_effect = list_side_effect
1353
1354         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1355             capture_stderr = StringIO()
1356             cwltool_logger = logging.getLogger('cwltool')
1357             stderr_logger = logging.StreamHandler(capture_stderr)
1358             cwltool_logger.addHandler(stderr_logger)
1359
1360             try:
1361                 exited = arvados_cwl.main(
1362                     ["--submit", "--no-wait", "--api=containers", "--debug",
1363                         "tests/wf/submit_wf.cwl", infile],
1364                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1365
1366                 self.assertEqual(exited, 1)
1367                 self.assertRegex(
1368                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1369                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1370             finally:
1371                 cwltool_logger.removeHandler(stderr_logger)
1372
1373     @mock.patch("arvados.collection.CollectionReader")
1374     @stubs()
1375     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1376         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1377         capture_stderr = StringIO()
1378
1379         cwltool_logger = logging.getLogger('cwltool')
1380         stderr_logger = logging.StreamHandler(capture_stderr)
1381         cwltool_logger.addHandler(stderr_logger)
1382
1383         exited = arvados_cwl.main(
1384             ["--submit", "--no-wait", "--api=containers", "--debug",
1385                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1386             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1387
1388         try:
1389             self.assertEqual(exited, 1)
1390             self.assertRegex(
1391                 capture_stderr.getvalue(),
1392                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1393         finally:
1394             cwltool_logger.removeHandler(stderr_logger)
1395
1396     @stubs(('submit_wf_process_properties.cwl', None))
1397     def test_submit_set_process_properties(self, stubs):
1398         exited = arvados_cwl.main(
1399             ["--submit", "--no-wait", "--api=containers", "--debug",
1400                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1401             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1402
1403         expect_container = copy.deepcopy(stubs.expect_container_spec)
1404
1405         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1406             {
1407                 "class": "http://arvados.org/cwl#ProcessProperties",
1408                 "processProperties": [
1409                     {"propertyName": "baz",
1410                      "propertyValue": "$(inputs.x.basename)"},
1411                     {"propertyName": "foo",
1412                      "propertyValue": "bar"},
1413                     {"propertyName": "quux",
1414                      "propertyValue": {
1415                          "q1": 1,
1416                          "q2": 2
1417                      }
1418                     }
1419                 ],
1420             },
1421             {
1422                 "acrContainerImage": "999999999999999999999999999999d3+99",
1423                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1424             }
1425         ]
1426         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1427         #    "arv": "http://arvados.org/cwl#"
1428         #}
1429
1430         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl"
1431
1432         expect_container["properties"].update({
1433             "baz": "blorp.txt",
1434             "foo": "bar",
1435             "quux": {
1436                 "q1": 1,
1437                 "q2": 2
1438             }
1439         })
1440
1441         stubs.api.container_requests().create.assert_called_with(
1442             body=JsonDiffMatcher(expect_container))
1443         self.assertEqual(stubs.capture_stdout.getvalue(),
1444                          stubs.expect_container_request_uuid + '\n')
1445         self.assertEqual(exited, 0)
1446
1447
1448     @stubs()
1449     def test_submit_enable_preemptible(self, stubs):
1450         exited = arvados_cwl.main(
1451             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1452                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1453             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1454
1455         expect_container = copy.deepcopy(stubs.expect_container_spec)
1456         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1457                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1458                                        '--eval-timeout=20', '--thread-count=0',
1459                                        '--enable-reuse', "--collection-cache-size=256",
1460                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1461                                        '--debug', '--on-error=continue',
1462                                        '--enable-preemptible',
1463                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1464
1465         stubs.api.container_requests().create.assert_called_with(
1466             body=JsonDiffMatcher(expect_container))
1467         self.assertEqual(stubs.capture_stdout.getvalue(),
1468                          stubs.expect_container_request_uuid + '\n')
1469         self.assertEqual(exited, 0)
1470
1471     @stubs()
1472     def test_submit_disable_preemptible(self, stubs):
1473         exited = arvados_cwl.main(
1474             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1475                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1477
1478         expect_container = copy.deepcopy(stubs.expect_container_spec)
1479         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1480                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1481                                        '--eval-timeout=20', '--thread-count=0',
1482                                        '--enable-reuse', "--collection-cache-size=256",
1483                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1484                                        '--debug', '--on-error=continue',
1485                                        '--disable-preemptible',
1486                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1487
1488         stubs.api.container_requests().create.assert_called_with(
1489             body=JsonDiffMatcher(expect_container))
1490         self.assertEqual(stubs.capture_stdout.getvalue(),
1491                          stubs.expect_container_request_uuid + '\n')
1492         self.assertEqual(exited, 0)
1493
1494     @stubs()
1495     def test_submit_container_prefer_cached_downloads(self, stubs):
1496         exited = arvados_cwl.main(
1497             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1498                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1499             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1500
1501         expect_container = copy.deepcopy(stubs.expect_container_spec)
1502         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1503                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1504                                        '--eval-timeout=20', '--thread-count=0',
1505                                        '--enable-reuse', "--collection-cache-size=256",
1506                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1507                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1508                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1509
1510         stubs.api.container_requests().create.assert_called_with(
1511             body=JsonDiffMatcher(expect_container))
1512         self.assertEqual(stubs.capture_stdout.getvalue(),
1513                          stubs.expect_container_request_uuid + '\n')
1514         self.assertEqual(exited, 0)
1515
1516     @stubs()
1517     def test_submit_container_varying_url_params(self, stubs):
1518         exited = arvados_cwl.main(
1519             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1520                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1521             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1522
1523         expect_container = copy.deepcopy(stubs.expect_container_spec)
1524         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1525                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1526                                        '--eval-timeout=20', '--thread-count=0',
1527                                        '--enable-reuse', "--collection-cache-size=256",
1528                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1529                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1530                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1531
1532         stubs.api.container_requests().create.assert_called_with(
1533             body=JsonDiffMatcher(expect_container))
1534         self.assertEqual(stubs.capture_stdout.getvalue(),
1535                          stubs.expect_container_request_uuid + '\n')
1536         self.assertEqual(exited, 0)
1537
1538
1539 class TestCreateWorkflow(unittest.TestCase):
1540     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1541     expect_workflow = StripYAMLComments(
1542         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1543     expect_workflow_altname = StripYAMLComments(
1544         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1545
1546     def setUp(self):
1547         cwltool.process._names = set()
1548         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1549
1550     def tearDown(self):
1551         root_logger = logging.getLogger('')
1552
1553         # Remove existing RuntimeStatusLoggingHandlers if they exist
1554         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1555         root_logger.handlers = handlers
1556
1557     @stubs()
1558     def test_create(self, stubs):
1559         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1560         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1561
1562         exited = arvados_cwl.main(
1563             ["--create-workflow", "--debug",
1564              "--api=containers",
1565              "--project-uuid", project_uuid,
1566              "--disable-git",
1567              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1568             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1569
1570         stubs.api.pipeline_templates().create.refute_called()
1571         stubs.api.container_requests().create.refute_called()
1572
1573         body = {
1574             "workflow": {
1575                 "owner_uuid": project_uuid,
1576                 "name": "submit_wf.cwl",
1577                 "description": "",
1578                 "definition": self.expect_workflow,
1579             }
1580         }
1581         stubs.api.workflows().create.assert_called_with(
1582             body=JsonDiffMatcher(body))
1583
1584         self.assertEqual(stubs.capture_stdout.getvalue(),
1585                          stubs.expect_workflow_uuid + '\n')
1586         self.assertEqual(exited, 0)
1587
1588     @stubs()
1589     def test_create_name(self, stubs):
1590         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1591         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1592
1593         exited = arvados_cwl.main(
1594             ["--create-workflow", "--debug",
1595              "--api=containers",
1596              "--project-uuid", project_uuid,
1597              "--name", "testing 123",
1598              "--disable-git",
1599              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1600             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1601
1602         stubs.api.pipeline_templates().create.refute_called()
1603         stubs.api.container_requests().create.refute_called()
1604
1605         body = {
1606             "workflow": {
1607                 "owner_uuid": project_uuid,
1608                 "name": "testing 123",
1609                 "description": "",
1610                 "definition": self.expect_workflow_altname,
1611             }
1612         }
1613         stubs.api.workflows().create.assert_called_with(
1614             body=JsonDiffMatcher(body))
1615
1616         self.assertEqual(stubs.capture_stdout.getvalue(),
1617                          stubs.expect_workflow_uuid + '\n')
1618         self.assertEqual(exited, 0)
1619
1620
1621     @stubs()
1622     def test_update(self, stubs):
1623         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1624         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1625
1626         exited = arvados_cwl.main(
1627             ["--update-workflow", self.existing_workflow_uuid,
1628              "--debug",
1629              "--disable-git",
1630              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1631             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1632
1633         body = {
1634             "workflow": {
1635                 "name": "submit_wf.cwl",
1636                 "description": "",
1637                 "definition": self.expect_workflow,
1638                 "owner_uuid": project_uuid
1639             }
1640         }
1641         stubs.api.workflows().update.assert_called_with(
1642             uuid=self.existing_workflow_uuid,
1643             body=JsonDiffMatcher(body))
1644         self.assertEqual(stubs.capture_stdout.getvalue(),
1645                          self.existing_workflow_uuid + '\n')
1646         self.assertEqual(exited, 0)
1647
1648
1649     @stubs()
1650     def test_update_name(self, stubs):
1651         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1652         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1653
1654         exited = arvados_cwl.main(
1655             ["--update-workflow", self.existing_workflow_uuid,
1656              "--debug", "--name", "testing 123",
1657              "--disable-git",
1658              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1659             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1660
1661         body = {
1662             "workflow": {
1663                 "name": "testing 123",
1664                 "description": "",
1665                 "definition": self.expect_workflow_altname,
1666                 "owner_uuid": project_uuid
1667             }
1668         }
1669         stubs.api.workflows().update.assert_called_with(
1670             uuid=self.existing_workflow_uuid,
1671             body=JsonDiffMatcher(body))
1672         self.assertEqual(stubs.capture_stdout.getvalue(),
1673                          self.existing_workflow_uuid + '\n')
1674         self.assertEqual(exited, 0)
1675
1676     @stubs()
1677     def test_create_collection_per_tool(self, stubs):
1678         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1679         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1680
1681         exited = arvados_cwl.main(
1682             ["--create-workflow", "--debug",
1683              "--api=containers",
1684              "--project-uuid", project_uuid,
1685              "--disable-git",
1686              "tests/collection_per_tool/collection_per_tool.cwl"],
1687             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1688
1689         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1690         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1691
1692         body = {
1693             "workflow": {
1694                 "owner_uuid": project_uuid,
1695                 "name": "collection_per_tool.cwl",
1696                 "description": "",
1697                 "definition": expect_workflow,
1698             }
1699         }
1700         stubs.api.workflows().create.assert_called_with(
1701             body=JsonDiffMatcher(body))
1702
1703         self.assertEqual(stubs.capture_stdout.getvalue(),
1704                          stubs.expect_workflow_uuid + '\n')
1705         self.assertEqual(exited, 0)
1706
1707     @stubs()
1708     def test_create_with_imports(self, stubs):
1709         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1710         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1711
1712         exited = arvados_cwl.main(
1713             ["--create-workflow", "--debug",
1714              "--api=containers",
1715              "--project-uuid", project_uuid,
1716              "tests/wf/feddemo/feddemo.cwl"],
1717             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1718
1719         stubs.api.pipeline_templates().create.refute_called()
1720         stubs.api.container_requests().create.refute_called()
1721
1722         self.assertEqual(stubs.capture_stdout.getvalue(),
1723                          stubs.expect_workflow_uuid + '\n')
1724         self.assertEqual(exited, 0)
1725
1726     @stubs()
1727     def test_create_with_no_input(self, stubs):
1728         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1729         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1730
1731         exited = arvados_cwl.main(
1732             ["--create-workflow", "--debug",
1733              "--api=containers",
1734              "--project-uuid", project_uuid,
1735              "tests/wf/revsort/revsort.cwl"],
1736             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1737
1738         stubs.api.pipeline_templates().create.refute_called()
1739         stubs.api.container_requests().create.refute_called()
1740
1741         self.assertEqual(stubs.capture_stdout.getvalue(),
1742                          stubs.expect_workflow_uuid + '\n')
1743         self.assertEqual(exited, 0)