sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfdetails=('submit_wf.cwl', None)):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62
  63             wfname = wfdetails[0]
  64             wfpath = wfdetails[1]
  65
  66             stubs = Stubs()
  67             stubs.events = events
  68             stubs.keepdocker = keepdocker
  69
  70             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  71                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
  73                                  "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
  74                                  "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
  75                                  "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
  76                                  "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
  77                                  "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
  78                                  "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
  79
  80             determine_image_id.return_value = None
  81
  82             def putstub(p, **kwargs):
  83                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  84             keep_client1().put.side_effect = putstub
  85             keep_client1.put.side_effect = putstub
  86             keep_client2().put.side_effect = putstub
  87             keep_client2.put.side_effect = putstub
  88
  89             stubs.keep_client = keep_client2
  90             stubs.docker_images = {
  91                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  92                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  93                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  94                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  95             }
  96             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  97                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  98             stubs.keepdocker.side_effect = kd
  99
 100             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
 101             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
 102
 103             if sys.version_info[0] < 3:
 104                 stubs.capture_stdout = BytesIO()
 105             else:
 106                 stubs.capture_stdout = StringIO()
 107
 108             stubs.api = mock.MagicMock()
 109             stubs.api._rootDesc = get_rootDesc()
 110             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 111             stubs.api._rootDesc["revision"] = "20210628"
 112
 113             stubs.api.users().current().execute.return_value = {
 114                 "uuid": stubs.fake_user_uuid,
 115             }
 116             stubs.api.collections().list().execute.return_value = {"items": []}
 117             stubs.api.containers().current().execute.return_value = {
 118                 "uuid": stubs.fake_container_uuid,
 119             }
 120             stubs.api.config()["StorageClasses"].items.return_value = {
 121                 "default": {
 122                     "Default": True
 123                 }
 124             }.items()
 125
 126             class CollectionExecute(object):
 127                 def __init__(self, exe):
 128                     self.exe = exe
 129                 def execute(self, num_retries=None):
 130                     return self.exe
 131
 132             def collection_createstub(created_collections, body, ensure_unique_name=None):
 133                 mt = body["manifest_text"].encode('utf-8')
 134                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 135                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 136                 created_collections[uuid] = {
 137                     "uuid": uuid,
 138                     "portable_data_hash": pdh,
 139                     "manifest_text": mt.decode('utf-8')
 140                 }
 141                 return CollectionExecute(created_collections[uuid])
 142
 143             def collection_getstub(created_collections, uuid):
 144                 for v in viewvalues(created_collections):
 145                     if uuid in (v["uuid"], v["portable_data_hash"]):
 146                         return CollectionExecute(v)
 147
 148             created_collections = {
 149                 "99999999999999999999999999999998+99": {
 150                     "uuid": "",
 151                     "portable_data_hash": "99999999999999999999999999999998+99",
 152                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 153                 },
 154                 "99999999999999999999999999999997+99": {
 155                     "uuid": "",
 156                     "portable_data_hash": "99999999999999999999999999999997+99",
 157                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 158                 },
 159                 "99999999999999999999999999999994+99": {
 160                     "uuid": "",
 161                     "portable_data_hash": "99999999999999999999999999999994+99",
 162                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 163                 },
 164                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 165                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 166                     "portable_data_hash": "999999999999999999999999999999d3+99",
 167                     "manifest_text": ""
 168                 },
 169                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 170                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 171                     "portable_data_hash": "999999999999999999999999999999d4+99",
 172                     "manifest_text": ""
 173                 },
 174                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 175                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 176                     "portable_data_hash": "999999999999999999999999999999d5+99",
 177                     "manifest_text": ""
 178                 },
 179                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 180                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 181                     "portable_data_hash": "999999999999999999999999999999d6+99",
 182                     "manifest_text": ""
 183                 }
 184             }
 185             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 186             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 187
 188             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 189             stubs.api.container_requests().create().execute.return_value = {
 190                 "uuid": stubs.expect_container_request_uuid,
 191                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 192                 "state": "Queued"
 193             }
 194
 195             cwd = os.getcwd()
 196             filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
 197             with open(filepath) as f:
 198                 yaml = ruamel.yaml.YAML(typ='rt', pure=True)
 199                 expect_packed_workflow = yaml.load(f)
 200
 201             if wfpath is None:
 202                 wfpath = wfname
 203
 204             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 205             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 206             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 207
 208             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 209             expect_packed_workflow.update(stubs.git_info)
 210
 211             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 212
 213             step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 214             if wfname == wfpath:
 215                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 216             else:
 217                 container_name = wfname
 218
 219             expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
 220             expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
 221
 222             stubs.expect_container_spec = {
 223                 'priority': 500,
 224                 'mounts': {
 225                     '/var/spool/cwl': {
 226                         'writable': True,
 227                         'kind': 'collection'
 228                     },
 229                     '/var/lib/cwl/workflow.json': {
 230                         'content': expect_packed_workflow,
 231                         'kind': 'json'
 232                     },
 233                     'stdout': {
 234                         'path': '/var/spool/cwl/cwl.output.json',
 235                         'kind': 'file'
 236                     },
 237                     '/var/lib/cwl/cwl.input.json': {
 238                         'kind': 'json',
 239                         'content': {
 240                             'y': {
 241                                 'basename': '99999999999999999999999999999998+99',
 242                                 'location': 'keep:99999999999999999999999999999998+99',
 243                                 'class': 'Directory'},
 244                             'x': {
 245                                 'basename': u'blorp.txt',
 246                                 'class': 'File',
 247                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 248                                 "size": 16
 249                             },
 250                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 251                                 {'basename': 'renamed.txt',
 252                                  'class': 'File',
 253                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 254                                  'size': 0
 255                                 }
 256                             ]}
 257                         },
 258                         'kind': 'json'
 259                     }
 260                 },
 261                 'secret_mounts': {},
 262                 'state': 'Committed',
 263                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 264                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 265                             '--eval-timeout=20', '--thread-count=0',
 266                             '--enable-reuse', "--collection-cache-size=256",
 267                             '--output-name=Output from workflow '+container_name,
 268                             '--debug', '--on-error=continue',
 269                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 270                 'name': container_name,
 271                 'container_image': '999999999999999999999999999999d3+99',
 272                 'output_name': 'Output from workflow %s' % (container_name),
 273                 'output_path': '/var/spool/cwl',
 274                 'cwd': '/var/spool/cwl',
 275                 'runtime_constraints': {
 276                     'API': True,
 277                     'vcpus': 1,
 278                     'ram': (1024+256)*1024*1024
 279                 },
 280                 'properties': stubs.git_props,
 281                 'use_existing': False,
 282                 'secret_mounts': {}
 283             }
 284
 285             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 286             stubs.api.workflows().create().execute.return_value = {
 287                 "uuid": stubs.expect_workflow_uuid,
 288             }
 289             def update_mock(**kwargs):
 290                 stubs.updated_uuid = kwargs.get('uuid')
 291                 return mock.DEFAULT
 292             stubs.api.workflows().update.side_effect = update_mock
 293             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 294                 "uuid": stubs.updated_uuid,
 295             }
 296
 297             return func(self, stubs, *args, **kwargs)
 298         return wrapped
 299     return outer_wrapper
 300
 301 class TestSubmit(unittest.TestCase):
 302
 303     def setUp(self):
 304         cwltool.process._names = set()
 305         #arvados_cwl.arvdocker.arv_docker_clear_cache()
 306
 307     def tearDown(self):
 308         root_logger = logging.getLogger('')
 309
 310         # Remove existing RuntimeStatusLoggingHandlers if they exist
 311         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 312         root_logger.handlers = handlers
 313
 314     @mock.patch("time.sleep")
 315     @stubs()
 316     def test_submit_invalid_runner_ram(self, stubs, tm):
 317         exited = arvados_cwl.main(
 318             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 1)
 322
 323
 324     @stubs()
 325     def test_submit_container(self, stubs):
 326         exited = arvados_cwl.main(
 327             ["--submit", "--no-wait", "--api=containers", "--debug",
 328                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 329             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 330
 331         stubs.api.collections().create.assert_has_calls([
 332             mock.call(body=JsonDiffMatcher({
 333                 'manifest_text':
 334                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 335                 'replication_desired': None,
 336                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 337             }), ensure_unique_name=False),
 338             mock.call(body=JsonDiffMatcher({
 339                 'manifest_text':
 340                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 341                 'replication_desired': None,
 342                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 343             }), ensure_unique_name=False),
 344             ])
 345
 346         expect_container = copy.deepcopy(stubs.expect_container_spec)
 347         stubs.api.container_requests().create.assert_called_with(
 348             body=JsonDiffMatcher(expect_container))
 349         self.assertEqual(stubs.capture_stdout.getvalue(),
 350                          stubs.expect_container_request_uuid + '\n')
 351         self.assertEqual(exited, 0)
 352
 353
 354     @stubs()
 355     def test_submit_container_tool(self, stubs):
 356         # test for issue #16139
 357         exited = arvados_cwl.main(
 358             ["--submit", "--no-wait", "--api=containers", "--debug",
 359                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 360             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 361
 362         self.assertEqual(stubs.capture_stdout.getvalue(),
 363                          stubs.expect_container_request_uuid + '\n')
 364         self.assertEqual(exited, 0)
 365
 366     @stubs()
 367     def test_submit_container_no_reuse(self, stubs):
 368         exited = arvados_cwl.main(
 369             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 370                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 371             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 372
 373         expect_container = copy.deepcopy(stubs.expect_container_spec)
 374         expect_container["command"] = [
 375             'arvados-cwl-runner', '--local', '--api=containers',
 376             '--no-log-timestamps', '--disable-validate', '--disable-color',
 377             '--eval-timeout=20', '--thread-count=0',
 378             '--disable-reuse', "--collection-cache-size=256",
 379             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 380             '--debug', '--on-error=continue',
 381             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 382         expect_container["use_existing"] = False
 383
 384         stubs.api.container_requests().create.assert_called_with(
 385             body=JsonDiffMatcher(expect_container))
 386         self.assertEqual(stubs.capture_stdout.getvalue(),
 387                          stubs.expect_container_request_uuid + '\n')
 388         self.assertEqual(exited, 0)
 389
 390     @stubs(('submit_wf_no_reuse.cwl', None))
 391     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 392         exited = arvados_cwl.main(
 393             ["--submit", "--no-wait", "--api=containers", "--debug",
 394              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 395             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 396         self.assertEqual(exited, 0)
 397
 398         expect_container = copy.deepcopy(stubs.expect_container_spec)
 399         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 400         expect_container["use_existing"] = False
 401         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 402             {
 403                 "class": "WorkReuse",
 404                 "enableReuse": False,
 405             },
 406                 {
 407                     "acrContainerImage": "999999999999999999999999999999d3+99",
 408                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 409                 }
 410         ]
 411         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
 412
 413         stubs.api.container_requests().create.assert_called_with(
 414             body=JsonDiffMatcher(expect_container))
 415         self.assertEqual(stubs.capture_stdout.getvalue(),
 416                          stubs.expect_container_request_uuid + '\n')
 417
 418
 419     @stubs()
 420     def test_submit_container_on_error(self, stubs):
 421         exited = arvados_cwl.main(
 422             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 423                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 424             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 425
 426         expect_container = copy.deepcopy(stubs.expect_container_spec)
 427         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 428                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 429                                        '--eval-timeout=20', '--thread-count=0',
 430                                        '--enable-reuse', "--collection-cache-size=256",
 431                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 432                                        '--debug', '--on-error=stop',
 433                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 434
 435         stubs.api.container_requests().create.assert_called_with(
 436             body=JsonDiffMatcher(expect_container))
 437         self.assertEqual(stubs.capture_stdout.getvalue(),
 438                          stubs.expect_container_request_uuid + '\n')
 439         self.assertEqual(exited, 0)
 440
 441     @stubs()
 442     def test_submit_container_output_name(self, stubs):
 443         output_name = "test_output_name"
 444
 445         exited = arvados_cwl.main(
 446             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 447                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 448             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 449
 450         expect_container = copy.deepcopy(stubs.expect_container_spec)
 451         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 452                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 453                                        '--eval-timeout=20', '--thread-count=0',
 454                                        '--enable-reuse', "--collection-cache-size=256",
 455                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 456                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 457         expect_container["output_name"] = output_name
 458
 459         stubs.api.container_requests().create.assert_called_with(
 460             body=JsonDiffMatcher(expect_container))
 461         self.assertEqual(stubs.capture_stdout.getvalue(),
 462                          stubs.expect_container_request_uuid + '\n')
 463         self.assertEqual(exited, 0)
 464
 465     @stubs()
 466     def test_submit_storage_classes(self, stubs):
 467         exited = arvados_cwl.main(
 468             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 469                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 470             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 471
 472         expect_container = copy.deepcopy(stubs.expect_container_spec)
 473         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 474                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 475                                        '--eval-timeout=20', '--thread-count=0',
 476                                        '--enable-reuse', "--collection-cache-size=256",
 477                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 478                                        "--debug",
 479                                        "--storage-classes=foo", '--on-error=continue',
 480                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 481
 482         stubs.api.container_requests().create.assert_called_with(
 483             body=JsonDiffMatcher(expect_container))
 484         self.assertEqual(stubs.capture_stdout.getvalue(),
 485                          stubs.expect_container_request_uuid + '\n')
 486         self.assertEqual(exited, 0)
 487
 488     @stubs()
 489     def test_submit_multiple_storage_classes(self, stubs):
 490         exited = arvados_cwl.main(
 491             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 492                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 493             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 494
 495         expect_container = copy.deepcopy(stubs.expect_container_spec)
 496         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 497                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 498                                        '--eval-timeout=20', '--thread-count=0',
 499                                        '--enable-reuse', "--collection-cache-size=256",
 500                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 501                                        "--debug",
 502                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 503                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 504
 505         stubs.api.container_requests().create.assert_called_with(
 506             body=JsonDiffMatcher(expect_container))
 507         self.assertEqual(stubs.capture_stdout.getvalue(),
 508                          stubs.expect_container_request_uuid + '\n')
 509         self.assertEqual(exited, 0)
 510
 511     @mock.patch("cwltool.task_queue.TaskQueue")
 512     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 513     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 514     @stubs()
 515     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 516         final_output_c = arvados.collection.Collection()
 517         make_output.return_value = ({},final_output_c)
 518
 519         def set_final_output(job_order, output_callback, runtimeContext):
 520             output_callback({"out": "zzzzz"}, "success")
 521             return []
 522         job.side_effect = set_final_output
 523
 524         exited = arvados_cwl.main(
 525             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 526                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 527             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 528
 529         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 530         self.assertEqual(exited, 0)
 531
 532     @mock.patch("cwltool.task_queue.TaskQueue")
 533     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 534     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 535     @stubs()
 536     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 537         final_output_c = arvados.collection.Collection()
 538         make_output.return_value = ({},final_output_c)
 539         stubs.api.config().get.return_value = {"default": {"Default": True}}
 540
 541         def set_final_output(job_order, output_callback, runtimeContext):
 542             output_callback({"out": "zzzzz"}, "success")
 543             return []
 544         job.side_effect = set_final_output
 545
 546         exited = arvados_cwl.main(
 547             ["--debug", "--local", "--disable-git",
 548                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 549             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 550
 551         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 552         self.assertEqual(exited, 0)
 553
 554     @mock.patch("cwltool.task_queue.TaskQueue")
 555     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 556     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 557     @stubs()
 558     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 559         final_output_c = arvados.collection.Collection()
 560         make_output.return_value = ({},final_output_c)
 561
 562         def set_final_output(job_order, output_callback, runtimeContext):
 563             output_callback({"out": "zzzzz"}, "success")
 564             return []
 565         job.side_effect = set_final_output
 566
 567         exited = arvados_cwl.main(
 568             ["--debug", "--local", "--disable-git",
 569                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 570             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 571
 572         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 573         self.assertEqual(exited, 0)
 574
 575     @stubs()
 576     def test_submit_container_output_ttl(self, stubs):
 577         exited = arvados_cwl.main(
 578             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 579                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 580             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 581
 582         expect_container = copy.deepcopy(stubs.expect_container_spec)
 583         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 584                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 585                                        '--eval-timeout=20', '--thread-count=0',
 586                                        '--enable-reuse', "--collection-cache-size=256",
 587                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 588                                        '--debug',
 589                                        '--on-error=continue',
 590                                        "--intermediate-output-ttl=3600",
 591                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 592
 593         stubs.api.container_requests().create.assert_called_with(
 594             body=JsonDiffMatcher(expect_container))
 595         self.assertEqual(stubs.capture_stdout.getvalue(),
 596                          stubs.expect_container_request_uuid + '\n')
 597         self.assertEqual(exited, 0)
 598
 599     @stubs()
 600     def test_submit_container_trash_intermediate(self, stubs):
 601         exited = arvados_cwl.main(
 602             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 603                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 604             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 605
 606
 607         expect_container = copy.deepcopy(stubs.expect_container_spec)
 608         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 609                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 610                                        '--eval-timeout=20', '--thread-count=0',
 611                                        '--enable-reuse', "--collection-cache-size=256",
 612                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 613                                        '--debug', '--on-error=continue',
 614                                        "--trash-intermediate",
 615                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 616
 617         stubs.api.container_requests().create.assert_called_with(
 618             body=JsonDiffMatcher(expect_container))
 619         self.assertEqual(stubs.capture_stdout.getvalue(),
 620                          stubs.expect_container_request_uuid + '\n')
 621         self.assertEqual(exited, 0)
 622
 623     @stubs()
 624     def test_submit_container_output_tags(self, stubs):
 625         output_tags = "tag0,tag1,tag2"
 626
 627         exited = arvados_cwl.main(
 628             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 629                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 630             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 631
 632         expect_container = copy.deepcopy(stubs.expect_container_spec)
 633         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 634                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 635                                        '--eval-timeout=20', '--thread-count=0',
 636                                        '--enable-reuse', "--collection-cache-size=256",
 637                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 638                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 639                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 640
 641         stubs.api.container_requests().create.assert_called_with(
 642             body=JsonDiffMatcher(expect_container))
 643         self.assertEqual(stubs.capture_stdout.getvalue(),
 644                          stubs.expect_container_request_uuid + '\n')
 645         self.assertEqual(exited, 0)
 646
 647     @stubs()
 648     def test_submit_container_runner_ram(self, stubs):
 649         exited = arvados_cwl.main(
 650             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 651                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 652             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 653
 654         expect_container = copy.deepcopy(stubs.expect_container_spec)
 655         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 656         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 657                 {
 658                     "acrContainerImage": "999999999999999999999999999999d3+99",
 659                     "class": "http://arvados.org/cwl#WorkflowRunnerResources",
 660                     "ramMin": 2048
 661                 }
 662         ]
 663
 664         stubs.api.container_requests().create.assert_called_with(
 665             body=JsonDiffMatcher(expect_container))
 666         self.assertEqual(stubs.capture_stdout.getvalue(),
 667                          stubs.expect_container_request_uuid + '\n')
 668         self.assertEqual(exited, 0)
 669
 670     @mock.patch("arvados.collection.CollectionReader")
 671     @mock.patch("time.sleep")
 672     @stubs()
 673     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 674         collectionReader().exists.return_value = True
 675         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 676         exited = arvados_cwl.main(
 677             ["--submit", "--no-wait", "--api=containers", "--debug",
 678              "tests/wf/submit_keepref_wf.cwl"],
 679             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 680         self.assertEqual(exited, 0)
 681
 682     @mock.patch("arvados.collection.CollectionReader")
 683     @mock.patch("time.sleep")
 684     @stubs()
 685     def test_submit_keepref(self, stubs, tm, reader):
 686         with open("tests/wf/expect_arvworkflow.cwl") as f:
 687             reader().open().__enter__().read.return_value = f.read()
 688
 689         exited = arvados_cwl.main(
 690             ["--submit", "--no-wait", "--api=containers", "--debug",
 691              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 692             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 693
 694         expect_container = {
 695             'priority': 500,
 696             'mounts': {
 697                 '/var/spool/cwl': {
 698                     'writable': True,
 699                     'kind': 'collection'
 700                 },
 701                 'stdout': {
 702                     'path': '/var/spool/cwl/cwl.output.json',
 703                     'kind': 'file'
 704                 },
 705                 '/var/lib/cwl/workflow': {
 706                     'portable_data_hash': '99999999999999999999999999999994+99',
 707                     'kind': 'collection'
 708                 },
 709                 '/var/lib/cwl/cwl.input.json': {
 710                     'content': {
 711                         'x': 'XxX'
 712                     },
 713                     'kind': 'json'
 714                 }
 715             }, 'state': 'Committed',
 716             'output_path': '/var/spool/cwl',
 717             'name': 'expect_arvworkflow.cwl#main',
 718             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 719             'container_image': '999999999999999999999999999999d3+99',
 720             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 721                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 722                         '--eval-timeout=20', '--thread-count=0',
 723                         '--enable-reuse', "--collection-cache-size=256",
 724                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 725                         '--debug', '--on-error=continue',
 726                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 727             'cwd': '/var/spool/cwl',
 728             'runtime_constraints': {
 729                 'API': True,
 730                 'vcpus': 1,
 731                 'ram': 1342177280
 732             },
 733             'use_existing': False,
 734             'properties': {},
 735             'secret_mounts': {}
 736         }
 737
 738         stubs.api.container_requests().create.assert_called_with(
 739             body=JsonDiffMatcher(expect_container))
 740         self.assertEqual(stubs.capture_stdout.getvalue(),
 741                          stubs.expect_container_request_uuid + '\n')
 742         self.assertEqual(exited, 0)
 743
 744     @mock.patch("time.sleep")
 745     @stubs()
 746     def test_submit_arvworkflow(self, stubs, tm):
 747         with open("tests/wf/expect_arvworkflow.cwl") as f:
 748             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 749
 750         exited = arvados_cwl.main(
 751             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 752              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 753             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 754
 755         expect_container = {
 756             'priority': 500,
 757             'mounts': {
 758                 '/var/spool/cwl': {
 759                     'writable': True,
 760                     'kind': 'collection'
 761                 },
 762                 'stdout': {
 763                     'path': '/var/spool/cwl/cwl.output.json',
 764                     'kind': 'file'
 765                 },
 766                 '/var/lib/cwl/workflow.json': {
 767                     'kind': 'json',
 768                     'content': {
 769                         'cwlVersion': 'v1.0',
 770                         'label': 'a test workflow',
 771                         '$graph': [
 772                             {
 773                                 'id': '#main',
 774                                 'inputs': [
 775                                     {'type': 'string', 'id': '#main/x'}
 776                                 ],
 777                                 'steps': [
 778                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 779                                      'run': '#submit_tool.cwl',
 780                                      'id': '#main/step1',
 781                                      'out': []}
 782                                 ],
 783                                 'class': 'Workflow',
 784                                 'outputs': []
 785                             },
 786                             {
 787                                 'inputs': [
 788                                     {
 789                                         'inputBinding': {'position': 1},
 790                                         'type': 'string',
 791                                         'id': '#submit_tool.cwl/x'}
 792                                 ],
 793                                 'requirements': [
 794                                     {
 795                                         'dockerPull': 'debian:buster-slim',
 796                                         'class': 'DockerRequirement'
 797                                     }
 798                                 ],
 799                                 'id': '#submit_tool.cwl',
 800                                 'outputs': [],
 801                                 'baseCommand': 'cat',
 802                                 'class': 'CommandLineTool'
 803                             }
 804                         ]
 805                     }
 806                 },
 807                 '/var/lib/cwl/cwl.input.json': {
 808                     'content': {
 809                         'x': 'XxX'
 810                     },
 811                     'kind': 'json'
 812                 }
 813             }, 'state': 'Committed',
 814             'output_path': '/var/spool/cwl',
 815             'name': 'a test workflow',
 816             'container_image': "999999999999999999999999999999d3+99",
 817             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 818                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 819                         '--eval-timeout=20', '--thread-count=0',
 820                         '--enable-reuse', "--collection-cache-size=256",
 821                         "--output-name=Output from workflow a test workflow",
 822                         '--debug', '--on-error=continue',
 823                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 824             'output_name': 'Output from workflow a test workflow',
 825             'cwd': '/var/spool/cwl',
 826             'runtime_constraints': {
 827                 'API': True,
 828                 'vcpus': 1,
 829                 'ram': 1342177280
 830             },
 831             'use_existing': False,
 832             'properties': {
 833                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 834             },
 835             'secret_mounts': {}
 836         }
 837
 838         stubs.api.container_requests().create.assert_called_with(
 839             body=JsonDiffMatcher(expect_container))
 840         self.assertEqual(stubs.capture_stdout.getvalue(),
 841                          stubs.expect_container_request_uuid + '\n')
 842         self.assertEqual(exited, 0)
 843
 844     @stubs(('hello container 123', 'submit_wf.cwl'))
 845     def test_submit_container_name(self, stubs):
 846         exited = arvados_cwl.main(
 847             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 848                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 849             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 850
 851         expect_container = copy.deepcopy(stubs.expect_container_spec)
 852
 853         stubs.api.container_requests().create.assert_called_with(
 854             body=JsonDiffMatcher(expect_container))
 855         self.assertEqual(stubs.capture_stdout.getvalue(),
 856                          stubs.expect_container_request_uuid + '\n')
 857         self.assertEqual(exited, 0)
 858
 859     @stubs()
 860     def test_submit_missing_input(self, stubs):
 861         exited = arvados_cwl.main(
 862             ["--submit", "--no-wait", "--api=containers", "--debug",
 863              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 864             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 865         self.assertEqual(exited, 0)
 866
 867         exited = arvados_cwl.main(
 868             ["--submit", "--no-wait", "--api=containers", "--debug",
 869              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 870             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 871         self.assertEqual(exited, 1)
 872
 873     @stubs()
 874     def test_submit_container_project(self, stubs):
 875         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 876         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 877         exited = arvados_cwl.main(
 878             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 879                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 880             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 881
 882         expect_container = copy.deepcopy(stubs.expect_container_spec)
 883         expect_container["owner_uuid"] = project_uuid
 884         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 885                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 886                                        "--eval-timeout=20", "--thread-count=0",
 887                                        '--enable-reuse', "--collection-cache-size=256",
 888                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 889                                        '--debug',
 890                                        '--on-error=continue',
 891                                        '--project-uuid='+project_uuid,
 892                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 893
 894         stubs.api.container_requests().create.assert_called_with(
 895             body=JsonDiffMatcher(expect_container))
 896         self.assertEqual(stubs.capture_stdout.getvalue(),
 897                          stubs.expect_container_request_uuid + '\n')
 898         self.assertEqual(exited, 0)
 899
 900     @stubs()
 901     def test_submit_container_eval_timeout(self, stubs):
 902         exited = arvados_cwl.main(
 903             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 904                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 905             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 906
 907         expect_container = copy.deepcopy(stubs.expect_container_spec)
 908         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 909                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 910                                        '--eval-timeout=60.0', '--thread-count=0',
 911                                        '--enable-reuse', "--collection-cache-size=256",
 912                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 913                                        '--debug', '--on-error=continue',
 914                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 915
 916         stubs.api.container_requests().create.assert_called_with(
 917             body=JsonDiffMatcher(expect_container))
 918         self.assertEqual(stubs.capture_stdout.getvalue(),
 919                          stubs.expect_container_request_uuid + '\n')
 920         self.assertEqual(exited, 0)
 921
 922     @stubs()
 923     def test_submit_container_collection_cache(self, stubs):
 924         exited = arvados_cwl.main(
 925             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 926                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 927             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 928
 929         expect_container = copy.deepcopy(stubs.expect_container_spec)
 930         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 931                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 932                                        '--eval-timeout=20', '--thread-count=0',
 933                                        '--enable-reuse', "--collection-cache-size=500",
 934                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 935                                        '--debug', '--on-error=continue',
 936                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 937         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
 938
 939         stubs.api.container_requests().create.assert_called_with(
 940             body=JsonDiffMatcher(expect_container))
 941         self.assertEqual(stubs.capture_stdout.getvalue(),
 942                          stubs.expect_container_request_uuid + '\n')
 943         self.assertEqual(exited, 0)
 944
 945     @stubs()
 946     def test_submit_container_thread_count(self, stubs):
 947         exited = arvados_cwl.main(
 948             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
 949                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 950             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 951
 952         expect_container = copy.deepcopy(stubs.expect_container_spec)
 953         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 954                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 955                                        '--eval-timeout=20', '--thread-count=20',
 956                                        '--enable-reuse', "--collection-cache-size=256",
 957                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 958                                        '--debug', '--on-error=continue',
 959                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 960
 961         stubs.api.container_requests().create.assert_called_with(
 962             body=JsonDiffMatcher(expect_container))
 963         self.assertEqual(stubs.capture_stdout.getvalue(),
 964                          stubs.expect_container_request_uuid + '\n')
 965         self.assertEqual(exited, 0)
 966
 967     @stubs()
 968     def test_submit_container_runner_image(self, stubs):
 969         exited = arvados_cwl.main(
 970             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
 971                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 972             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 973
 974         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 975         stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
 976                 {
 977                     "acrContainerImage": "999999999999999999999999999999d5+99",
 978                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
 979                 }
 980         ]
 981
 982         expect_container = copy.deepcopy(stubs.expect_container_spec)
 983         stubs.api.container_requests().create.assert_called_with(
 984             body=JsonDiffMatcher(expect_container))
 985         self.assertEqual(stubs.capture_stdout.getvalue(),
 986                          stubs.expect_container_request_uuid + '\n')
 987         self.assertEqual(exited, 0)
 988
 989     @stubs()
 990     def test_submit_priority(self, stubs):
 991         exited = arvados_cwl.main(
 992             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
 993                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 994             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 995
 996         stubs.expect_container_spec["priority"] = 669
 997
 998         expect_container = copy.deepcopy(stubs.expect_container_spec)
 999         stubs.api.container_requests().create.assert_called_with(
1000             body=JsonDiffMatcher(expect_container))
1001         self.assertEqual(stubs.capture_stdout.getvalue(),
1002                          stubs.expect_container_request_uuid + '\n')
1003         self.assertEqual(exited, 0)
1004
1005     @stubs(('submit_wf_runner_resources.cwl', None))
1006     def test_submit_wf_runner_resources(self, stubs):
1007         exited = arvados_cwl.main(
1008             ["--submit", "--no-wait", "--api=containers", "--debug",
1009                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1010             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1011
1012         expect_container = copy.deepcopy(stubs.expect_container_spec)
1013         expect_container["runtime_constraints"] = {
1014             "API": True,
1015             "vcpus": 2,
1016             "ram": (2000+512) * 2**20
1017         }
1018         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1019             {
1020                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1021                 "acrContainerImage": "999999999999999999999999999999d3+99",
1022                 "coresMin": 2,
1023                 "ramMin": 2000,
1024                 "keep_cache": 512
1025             }
1026         ]
1027         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1028         #    "arv": "http://arvados.org/cwl#",
1029         #}
1030         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1031         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
1032
1033         stubs.api.container_requests().create.assert_called_with(
1034             body=JsonDiffMatcher(expect_container))
1035         self.assertEqual(stubs.capture_stdout.getvalue(),
1036                          stubs.expect_container_request_uuid + '\n')
1037         self.assertEqual(exited, 0)
1038
1039     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1040     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1041     @mock.patch("arvados.api")
1042     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1043         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1044
1045         arvrunner = mock.MagicMock()
1046         arvrunner.project_uuid = ""
1047         api.return_value = mock.MagicMock()
1048         arvrunner.api = api.return_value
1049         arvrunner.runtimeContext.match_local_docker = False
1050         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1051                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1052                                                                         "link_class": "docker_image_repo+tag",
1053                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1054                                                                         "owner_uuid": "",
1055                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1056                                                             {"items": [{"created_at": "",
1057                                                                         "head_uuid": "",
1058                                                                         "link_class": "docker_image_hash",
1059                                                                         "name": "123456",
1060                                                                         "owner_uuid": "",
1061                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1062                                                             {"items": [{"created_at": "",
1063                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1064                                                                         "link_class": "docker_image_repo+tag",
1065                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1066                                                                         "owner_uuid": "",
1067                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1068                                                             {"items": [{"created_at": "",
1069                                                                         "head_uuid": "",
1070                                                                         "link_class": "docker_image_hash",
1071                                                                         "name": "123456",
1072                                                                         "owner_uuid": "",
1073                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1074         )
1075         find_one_image_hash.return_value = "123456"
1076
1077         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1078                                                                               "owner_uuid": "",
1079                                                                               "manifest_text": "",
1080                                                                               "properties": ""
1081                                                                               }], "items_available": 1, "offset": 0},
1082                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1083                                                                               "owner_uuid": "",
1084                                                                               "manifest_text": "",
1085                                                                               "properties": ""
1086                                                                           }], "items_available": 1, "offset": 0})
1087         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1088         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1089                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1090
1091         self.assertEqual("9999999999999999999999999999999b+99",
1092                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1093
1094
1095     @stubs()
1096     def test_submit_secrets(self, stubs):
1097         exited = arvados_cwl.main(
1098             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
1099                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1100             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1101
1102         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1103         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1104
1105         expect_container = {
1106             "command": [
1107                 "arvados-cwl-runner",
1108                 "--local",
1109                 "--api=containers",
1110                 "--no-log-timestamps",
1111                 "--disable-validate",
1112                 "--disable-color",
1113                 "--eval-timeout=20",
1114                 '--thread-count=0',
1115                 "--enable-reuse",
1116                 "--collection-cache-size=256",
1117                 "--output-name=Output from workflow secret_wf.cwl",
1118                 "--debug",
1119                 "--on-error=continue",
1120                 "/var/lib/cwl/workflow.json#main",
1121                 "/var/lib/cwl/cwl.input.json"
1122             ],
1123             "container_image": "999999999999999999999999999999d3+99",
1124             "cwd": "/var/spool/cwl",
1125             "mounts": {
1126                 "/var/lib/cwl/cwl.input.json": {
1127                     "content": {
1128                         "pw": {
1129                             "$include": "/secrets/s0"
1130                         }
1131                     },
1132                     "kind": "json"
1133                 },
1134                 "/var/lib/cwl/workflow.json": {
1135                     "content": {
1136                         "$graph": [
1137                             {
1138                                 "class": "Workflow",
1139                                 "hints": [
1140                                     {
1141                                     "class": "DockerRequirement",
1142                                     "dockerPull": "debian:buster-slim",
1143                                     "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1144                                     },
1145                                     {
1146                                         "class": "http://commonwl.org/cwltool#Secrets",
1147                                         "secrets": [
1148                                             "#main/pw"
1149                                         ]
1150                                     },
1151                                     {
1152                                         "acrContainerImage": "999999999999999999999999999999d3+99",
1153                                         "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1154                                     }
1155                                 ],
1156                                 "id": "#main",
1157                                 "inputs": [
1158                                     {
1159                                         "id": "#main/pw",
1160                                         "type": "string"
1161                                     }
1162                                 ],
1163                                 "outputs": [
1164                                     {
1165                                         "id": "#main/out",
1166                                         "outputSource": "#main/step/out",
1167                                         "type": "File"
1168                                     }
1169                                 ],
1170                                 "requirements": [
1171                                     {
1172                                         "class": "SubworkflowFeatureRequirement"
1173                                     }
1174                                 ],
1175                                 "steps": [
1176                                     {
1177                                         "id": "#main/secret_wf.cwl",
1178                                         "in": [
1179                                             {
1180                                                 "id": "#main/step/pw",
1181                                                 "source": "#main/pw"
1182                                             }
1183                                         ],
1184                                         "label": "secret_wf.cwl",
1185                                         "out": [
1186                                             {"id": "#main/step/out"}
1187                                         ],
1188                                         "run": "keep:7628e49da34b93de9f4baf08a6212817+247/secret_wf.cwl"
1189                                     }
1190                                 ]
1191                             }
1192                         ],
1193                         "cwlVersion": "v1.2"
1194                     },
1195                     "kind": "json"
1196                 },
1197                 "/var/spool/cwl": {
1198                     "kind": "collection",
1199                     "writable": True
1200                 },
1201                 "stdout": {
1202                     "kind": "file",
1203                     "path": "/var/spool/cwl/cwl.output.json"
1204                 }
1205             },
1206             "name": "secret_wf.cwl",
1207             "output_name": "Output from workflow secret_wf.cwl",
1208             "output_path": "/var/spool/cwl",
1209             "priority": 500,
1210             "properties": {},
1211             "runtime_constraints": {
1212                 "API": True,
1213                 "ram": 1342177280,
1214                 "vcpus": 1
1215             },
1216             "secret_mounts": {
1217                 "/secrets/s0": {
1218                     "content": "blorp",
1219                     "kind": "text"
1220                 }
1221             },
1222             "state": "Committed",
1223             "use_existing": False
1224         }
1225
1226         stubs.api.container_requests().create.assert_called_with(
1227             body=JsonDiffMatcher(expect_container))
1228         self.assertEqual(stubs.capture_stdout.getvalue(),
1229                          stubs.expect_container_request_uuid + '\n')
1230         self.assertEqual(exited, 0)
1231
1232     @stubs()
1233     def test_submit_request_uuid(self, stubs):
1234         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1235         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1236
1237         stubs.api.container_requests().update().execute.return_value = {
1238             "uuid": stubs.expect_container_request_uuid,
1239             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1240             "state": "Queued"
1241         }
1242
1243         exited = arvados_cwl.main(
1244             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1245                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1246             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1247
1248         stubs.api.container_requests().update.assert_called_with(
1249             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1250         self.assertEqual(stubs.capture_stdout.getvalue(),
1251                          stubs.expect_container_request_uuid + '\n')
1252         self.assertEqual(exited, 0)
1253
1254     @stubs()
1255     def test_submit_container_cluster_id(self, stubs):
1256         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1257
1258         exited = arvados_cwl.main(
1259             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1260                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1261             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1262
1263         expect_container = copy.deepcopy(stubs.expect_container_spec)
1264
1265         stubs.api.container_requests().create.assert_called_with(
1266             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1267         self.assertEqual(stubs.capture_stdout.getvalue(),
1268                          stubs.expect_container_request_uuid + '\n')
1269         self.assertEqual(exited, 0)
1270
1271     @stubs()
1272     def test_submit_validate_cluster_id(self, stubs):
1273         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1274         exited = arvados_cwl.main(
1275             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1277             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1278         self.assertEqual(exited, 1)
1279
1280     @stubs()
1281     def test_submit_validate_project_uuid(self, stubs):
1282         # Fails with bad cluster prefix
1283         exited = arvados_cwl.main(
1284             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1285              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1286             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1287         self.assertEqual(exited, 1)
1288
1289         # Project lookup fails
1290         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1291         exited = arvados_cwl.main(
1292             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1293              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1294             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1295         self.assertEqual(exited, 1)
1296
1297         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1298         exited = arvados_cwl.main(
1299             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1300              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1301             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1302         self.assertEqual(exited, 0)
1303
1304
1305     @mock.patch("arvados.collection.CollectionReader")
1306     @stubs()
1307     def test_submit_uuid_inputs(self, stubs, collectionReader):
1308         collectionReader().exists.return_value = True
1309         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1310         def list_side_effect(**kwargs):
1311             m = mock.MagicMock()
1312             if "count" in kwargs:
1313                 m.execute.return_value = {"items": [
1314                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1315                 ]}
1316             else:
1317                 m.execute.return_value = {"items": []}
1318             return m
1319         stubs.api.collections().list.side_effect = list_side_effect
1320         collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
1321
1322         exited = arvados_cwl.main(
1323             ["--submit", "--no-wait", "--api=containers", "--debug",
1324                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1325             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1326
1327         expect_container = copy.deepcopy(stubs.expect_container_spec)
1328         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1329         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1330         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1331
1332         stubs.api.collections().list.assert_has_calls([
1333             mock.call(count='none',
1334                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1335                       select=['uuid', 'portable_data_hash'])])
1336         stubs.api.container_requests().create.assert_called_with(
1337             body=JsonDiffMatcher(expect_container))
1338         self.assertEqual(stubs.capture_stdout.getvalue(),
1339                          stubs.expect_container_request_uuid + '\n')
1340         self.assertEqual(exited, 0)
1341
1342     @stubs()
1343     def test_submit_mismatched_uuid_inputs(self, stubs):
1344         def list_side_effect(**kwargs):
1345             m = mock.MagicMock()
1346             if "count" in kwargs:
1347                 m.execute.return_value = {"items": [
1348                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1349                 ]}
1350             else:
1351                 m.execute.return_value = {"items": []}
1352             return m
1353         stubs.api.collections().list.side_effect = list_side_effect
1354
1355         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1356             capture_stderr = StringIO()
1357             cwltool_logger = logging.getLogger('cwltool')
1358             stderr_logger = logging.StreamHandler(capture_stderr)
1359             cwltool_logger.addHandler(stderr_logger)
1360
1361             try:
1362                 exited = arvados_cwl.main(
1363                     ["--submit", "--no-wait", "--api=containers", "--debug",
1364                         "tests/wf/submit_wf.cwl", infile],
1365                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1366
1367                 self.assertEqual(exited, 1)
1368                 self.assertRegex(
1369                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1370                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1371             finally:
1372                 cwltool_logger.removeHandler(stderr_logger)
1373
1374     @mock.patch("arvados.collection.CollectionReader")
1375     @stubs()
1376     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1377         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1378         capture_stderr = StringIO()
1379
1380         cwltool_logger = logging.getLogger('cwltool')
1381         stderr_logger = logging.StreamHandler(capture_stderr)
1382         cwltool_logger.addHandler(stderr_logger)
1383
1384         exited = arvados_cwl.main(
1385             ["--submit", "--no-wait", "--api=containers", "--debug",
1386                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1387             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1388
1389         try:
1390             self.assertEqual(exited, 1)
1391             self.assertRegex(
1392                 capture_stderr.getvalue(),
1393                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1394         finally:
1395             cwltool_logger.removeHandler(stderr_logger)
1396
1397     @stubs(('submit_wf_process_properties.cwl', None))
1398     def test_submit_set_process_properties(self, stubs):
1399         exited = arvados_cwl.main(
1400             ["--submit", "--no-wait", "--api=containers", "--debug",
1401                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1402             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1403
1404         expect_container = copy.deepcopy(stubs.expect_container_spec)
1405
1406         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
1407             {
1408                 "class": "http://arvados.org/cwl#ProcessProperties",
1409                 "processProperties": [
1410                     {"propertyName": "baz",
1411                      "propertyValue": "$(inputs.x.basename)"},
1412                     {"propertyName": "foo",
1413                      "propertyValue": "bar"},
1414                     {"propertyName": "quux",
1415                      "propertyValue": {
1416                          "q1": 1,
1417                          "q2": 2
1418                      }
1419                     }
1420                 ],
1421             },
1422             {
1423                 "acrContainerImage": "999999999999999999999999999999d3+99",
1424                 "class": "http://arvados.org/cwl#WorkflowRunnerResources"
1425             }
1426         ]
1427         #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1428         #    "arv": "http://arvados.org/cwl#"
1429         #}
1430
1431         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
1432
1433         expect_container["properties"].update({
1434             "baz": "blorp.txt",
1435             "foo": "bar",
1436             "quux": {
1437                 "q1": 1,
1438                 "q2": 2
1439             }
1440         })
1441
1442         stubs.api.container_requests().create.assert_called_with(
1443             body=JsonDiffMatcher(expect_container))
1444         self.assertEqual(stubs.capture_stdout.getvalue(),
1445                          stubs.expect_container_request_uuid + '\n')
1446         self.assertEqual(exited, 0)
1447
1448
1449     @stubs()
1450     def test_submit_enable_preemptible(self, stubs):
1451         exited = arvados_cwl.main(
1452             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1453                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1454             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1455
1456         expect_container = copy.deepcopy(stubs.expect_container_spec)
1457         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1458                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1459                                        '--eval-timeout=20', '--thread-count=0',
1460                                        '--enable-reuse', "--collection-cache-size=256",
1461                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1462                                        '--debug', '--on-error=continue',
1463                                        '--enable-preemptible',
1464                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1465
1466         stubs.api.container_requests().create.assert_called_with(
1467             body=JsonDiffMatcher(expect_container))
1468         self.assertEqual(stubs.capture_stdout.getvalue(),
1469                          stubs.expect_container_request_uuid + '\n')
1470         self.assertEqual(exited, 0)
1471
1472     @stubs()
1473     def test_submit_disable_preemptible(self, stubs):
1474         exited = arvados_cwl.main(
1475             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1476                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1477             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1478
1479         expect_container = copy.deepcopy(stubs.expect_container_spec)
1480         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1481                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1482                                        '--eval-timeout=20', '--thread-count=0',
1483                                        '--enable-reuse', "--collection-cache-size=256",
1484                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1485                                        '--debug', '--on-error=continue',
1486                                        '--disable-preemptible',
1487                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1488
1489         stubs.api.container_requests().create.assert_called_with(
1490             body=JsonDiffMatcher(expect_container))
1491         self.assertEqual(stubs.capture_stdout.getvalue(),
1492                          stubs.expect_container_request_uuid + '\n')
1493         self.assertEqual(exited, 0)
1494
1495     @stubs()
1496     def test_submit_container_prefer_cached_downloads(self, stubs):
1497         exited = arvados_cwl.main(
1498             ["--submit", "--no-wait", "--api=containers", "--debug", "--prefer-cached-downloads",
1499                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1500             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1501
1502         expect_container = copy.deepcopy(stubs.expect_container_spec)
1503         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1504                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1505                                        '--eval-timeout=20', '--thread-count=0',
1506                                        '--enable-reuse', "--collection-cache-size=256",
1507                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1508                                        '--debug', "--on-error=continue", '--prefer-cached-downloads',
1509                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1510
1511         stubs.api.container_requests().create.assert_called_with(
1512             body=JsonDiffMatcher(expect_container))
1513         self.assertEqual(stubs.capture_stdout.getvalue(),
1514                          stubs.expect_container_request_uuid + '\n')
1515         self.assertEqual(exited, 0)
1516
1517     @stubs()
1518     def test_submit_container_varying_url_params(self, stubs):
1519         exited = arvados_cwl.main(
1520             ["--submit", "--no-wait", "--api=containers", "--debug", "--varying-url-params", "KeyId,Signature",
1521                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1522             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1523
1524         expect_container = copy.deepcopy(stubs.expect_container_spec)
1525         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1526                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1527                                        '--eval-timeout=20', '--thread-count=0',
1528                                        '--enable-reuse', "--collection-cache-size=256",
1529                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1530                                        '--debug', "--on-error=continue", "--varying-url-params=KeyId,Signature",
1531                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1532
1533         stubs.api.container_requests().create.assert_called_with(
1534             body=JsonDiffMatcher(expect_container))
1535         self.assertEqual(stubs.capture_stdout.getvalue(),
1536                          stubs.expect_container_request_uuid + '\n')
1537         self.assertEqual(exited, 0)
1538
1539
1540 class TestCreateWorkflow(unittest.TestCase):
1541     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1542     expect_workflow = StripYAMLComments(
1543         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1544     expect_workflow_altname = StripYAMLComments(
1545         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1546
1547     def setUp(self):
1548         cwltool.process._names = set()
1549         #arvados_cwl.arvdocker.arv_docker_clear_cache()
1550
1551     def tearDown(self):
1552         root_logger = logging.getLogger('')
1553
1554         # Remove existing RuntimeStatusLoggingHandlers if they exist
1555         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1556         root_logger.handlers = handlers
1557
1558     @stubs()
1559     def test_create(self, stubs):
1560         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1561         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1562
1563         exited = arvados_cwl.main(
1564             ["--create-workflow", "--debug",
1565              "--api=containers",
1566              "--project-uuid", project_uuid,
1567              "--disable-git",
1568              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1569             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1570
1571         stubs.api.pipeline_templates().create.refute_called()
1572         stubs.api.container_requests().create.refute_called()
1573
1574         body = {
1575             "workflow": {
1576                 "owner_uuid": project_uuid,
1577                 "name": "submit_wf.cwl",
1578                 "description": "",
1579                 "definition": self.expect_workflow,
1580             }
1581         }
1582         stubs.api.workflows().create.assert_called_with(
1583             body=JsonDiffMatcher(body))
1584
1585         self.assertEqual(stubs.capture_stdout.getvalue(),
1586                          stubs.expect_workflow_uuid + '\n')
1587         self.assertEqual(exited, 0)
1588
1589     @stubs()
1590     def test_create_name(self, stubs):
1591         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1592         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1593
1594         exited = arvados_cwl.main(
1595             ["--create-workflow", "--debug",
1596              "--api=containers",
1597              "--project-uuid", project_uuid,
1598              "--name", "testing 123",
1599              "--disable-git",
1600              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1601             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1602
1603         stubs.api.pipeline_templates().create.refute_called()
1604         stubs.api.container_requests().create.refute_called()
1605
1606         body = {
1607             "workflow": {
1608                 "owner_uuid": project_uuid,
1609                 "name": "testing 123",
1610                 "description": "",
1611                 "definition": self.expect_workflow_altname,
1612             }
1613         }
1614         stubs.api.workflows().create.assert_called_with(
1615             body=JsonDiffMatcher(body))
1616
1617         self.assertEqual(stubs.capture_stdout.getvalue(),
1618                          stubs.expect_workflow_uuid + '\n')
1619         self.assertEqual(exited, 0)
1620
1621
1622     @stubs()
1623     def test_update(self, stubs):
1624         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1625         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1626
1627         exited = arvados_cwl.main(
1628             ["--update-workflow", self.existing_workflow_uuid,
1629              "--debug",
1630              "--disable-git",
1631              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1632             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1633
1634         body = {
1635             "workflow": {
1636                 "name": "submit_wf.cwl",
1637                 "description": "",
1638                 "definition": self.expect_workflow,
1639                 "owner_uuid": project_uuid
1640             }
1641         }
1642         stubs.api.workflows().update.assert_called_with(
1643             uuid=self.existing_workflow_uuid,
1644             body=JsonDiffMatcher(body))
1645         self.assertEqual(stubs.capture_stdout.getvalue(),
1646                          self.existing_workflow_uuid + '\n')
1647         self.assertEqual(exited, 0)
1648
1649
1650     @stubs()
1651     def test_update_name(self, stubs):
1652         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1653         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1654
1655         exited = arvados_cwl.main(
1656             ["--update-workflow", self.existing_workflow_uuid,
1657              "--debug", "--name", "testing 123",
1658              "--disable-git",
1659              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1660             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1661
1662         body = {
1663             "workflow": {
1664                 "name": "testing 123",
1665                 "description": "",
1666                 "definition": self.expect_workflow_altname,
1667                 "owner_uuid": project_uuid
1668             }
1669         }
1670         stubs.api.workflows().update.assert_called_with(
1671             uuid=self.existing_workflow_uuid,
1672             body=JsonDiffMatcher(body))
1673         self.assertEqual(stubs.capture_stdout.getvalue(),
1674                          self.existing_workflow_uuid + '\n')
1675         self.assertEqual(exited, 0)
1676
1677     @stubs()
1678     def test_create_collection_per_tool(self, stubs):
1679         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1680         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1681
1682         exited = arvados_cwl.main(
1683             ["--create-workflow", "--debug",
1684              "--api=containers",
1685              "--project-uuid", project_uuid,
1686              "--disable-git",
1687              "tests/collection_per_tool/collection_per_tool.cwl"],
1688             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1689
1690         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1691         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1692
1693         body = {
1694             "workflow": {
1695                 "owner_uuid": project_uuid,
1696                 "name": "collection_per_tool.cwl",
1697                 "description": "",
1698                 "definition": expect_workflow,
1699             }
1700         }
1701         stubs.api.workflows().create.assert_called_with(
1702             body=JsonDiffMatcher(body))
1703
1704         self.assertEqual(stubs.capture_stdout.getvalue(),
1705                          stubs.expect_workflow_uuid + '\n')
1706         self.assertEqual(exited, 0)
1707
1708     @stubs()
1709     def test_create_with_imports(self, stubs):
1710         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1711         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1712
1713         exited = arvados_cwl.main(
1714             ["--create-workflow", "--debug",
1715              "--api=containers",
1716              "--project-uuid", project_uuid,
1717              "tests/wf/feddemo/feddemo.cwl"],
1718             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1719
1720         stubs.api.pipeline_templates().create.refute_called()
1721         stubs.api.container_requests().create.refute_called()
1722
1723         self.assertEqual(stubs.capture_stdout.getvalue(),
1724                          stubs.expect_workflow_uuid + '\n')
1725         self.assertEqual(exited, 0)
1726
1727     @stubs()
1728     def test_create_with_no_input(self, stubs):
1729         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1730         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1731
1732         exited = arvados_cwl.main(
1733             ["--create-workflow", "--debug",
1734              "--api=containers",
1735              "--project-uuid", project_uuid,
1736              "tests/wf/revsort/revsort.cwl"],
1737             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1738
1739         stubs.api.pipeline_templates().create.refute_called()
1740         stubs.api.container_requests().create.refute_called()
1741
1742         self.assertEqual(stubs.capture_stdout.getvalue(),
1743                          stubs.expect_workflow_uuid + '\n')
1744         self.assertEqual(exited, 0)