sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml as yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfdetails=('submit_wf.cwl', None)):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62
  63             wfname = wfdetails[0]
  64             wfpath = wfdetails[1]
  65
  66             stubs = Stubs()
  67             stubs.events = events
  68             stubs.keepdocker = keepdocker
  69
  70             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  71                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  73
  74             determine_image_id.return_value = None
  75
  76             def putstub(p, **kwargs):
  77                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  78             keep_client1().put.side_effect = putstub
  79             keep_client1.put.side_effect = putstub
  80             keep_client2().put.side_effect = putstub
  81             keep_client2.put.side_effect = putstub
  82
  83             stubs.keep_client = keep_client2
  84             stubs.docker_images = {
  85                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  86                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  87                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  88                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  89             }
  90             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  91                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  92             stubs.keepdocker.side_effect = kd
  93
  94             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  95             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  96
  97             if sys.version_info[0] < 3:
  98                 stubs.capture_stdout = BytesIO()
  99             else:
 100                 stubs.capture_stdout = StringIO()
 101
 102             stubs.api = mock.MagicMock()
 103             stubs.api._rootDesc = get_rootDesc()
 104             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 105             stubs.api._rootDesc["revision"] = "20210628"
 106
 107             stubs.api.users().current().execute.return_value = {
 108                 "uuid": stubs.fake_user_uuid,
 109             }
 110             stubs.api.collections().list().execute.return_value = {"items": []}
 111             stubs.api.containers().current().execute.return_value = {
 112                 "uuid": stubs.fake_container_uuid,
 113             }
 114             stubs.api.config()["StorageClasses"].items.return_value = {
 115                 "default": {
 116                     "Default": True
 117                 }
 118             }.items()
 119
 120             class CollectionExecute(object):
 121                 def __init__(self, exe):
 122                     self.exe = exe
 123                 def execute(self, num_retries=None):
 124                     return self.exe
 125
 126             def collection_createstub(created_collections, body, ensure_unique_name=None):
 127                 mt = body["manifest_text"].encode('utf-8')
 128                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 129                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 130                 created_collections[uuid] = {
 131                     "uuid": uuid,
 132                     "portable_data_hash": pdh,
 133                     "manifest_text": mt.decode('utf-8')
 134                 }
 135                 return CollectionExecute(created_collections[uuid])
 136
 137             def collection_getstub(created_collections, uuid):
 138                 for v in viewvalues(created_collections):
 139                     if uuid in (v["uuid"], v["portable_data_hash"]):
 140                         return CollectionExecute(v)
 141
 142             created_collections = {
 143                 "99999999999999999999999999999998+99": {
 144                     "uuid": "",
 145                     "portable_data_hash": "99999999999999999999999999999998+99",
 146                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 147                 },
 148                 "99999999999999999999999999999997+99": {
 149                     "uuid": "",
 150                     "portable_data_hash": "99999999999999999999999999999997+99",
 151                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 152                 },
 153                 "99999999999999999999999999999994+99": {
 154                     "uuid": "",
 155                     "portable_data_hash": "99999999999999999999999999999994+99",
 156                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 157                 },
 158                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 159                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 160                     "portable_data_hash": "999999999999999999999999999999d3+99",
 161                     "manifest_text": ""
 162                 },
 163                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 164                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 165                     "portable_data_hash": "999999999999999999999999999999d4+99",
 166                     "manifest_text": ""
 167                 },
 168                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 169                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 170                     "portable_data_hash": "999999999999999999999999999999d5+99",
 171                     "manifest_text": ""
 172                 },
 173                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 174                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 175                     "portable_data_hash": "999999999999999999999999999999d6+99",
 176                     "manifest_text": ""
 177                 }
 178             }
 179             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 180             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 181
 182             stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 183             stubs.api.jobs().create().execute.return_value = {
 184                 "uuid": stubs.expect_job_uuid,
 185                 "state": "Queued",
 186             }
 187
 188             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 189             stubs.api.container_requests().create().execute.return_value = {
 190                 "uuid": stubs.expect_container_request_uuid,
 191                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 192                 "state": "Queued"
 193             }
 194
 195             stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 196             stubs.api.pipeline_templates().create().execute.return_value = {
 197                 "uuid": stubs.expect_pipeline_template_uuid,
 198             }
 199             stubs.expect_job_spec = {
 200                 'runtime_constraints': {
 201                     'docker_image': '999999999999999999999999999999d3+99',
 202                     'min_ram_mb_per_node': 1024
 203                 },
 204                 'script_parameters': {
 205                     'x': {
 206                         'basename': 'blorp.txt',
 207                         'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 208                         'class': 'File'
 209                     },
 210                     'y': {
 211                         'basename': '99999999999999999999999999999998+99',
 212                         'location': 'keep:99999999999999999999999999999998+99',
 213                         'class': 'Directory'
 214                     },
 215                     'z': {
 216                         'basename': 'anonymous',
 217                         "listing": [{
 218                             "basename": "renamed.txt",
 219                             "class": "File",
 220                             "location": "keep:99999999999999999999999999999998+99/file1.txt",
 221                             "size": 0
 222                         }],
 223                         'class': 'Directory'
 224                     },
 225                     'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 226                 },
 227                 'repository': 'arvados',
 228                 'script_version': 'master',
 229                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 230                 'script': 'cwl-runner'
 231             }
 232             stubs.pipeline_component = stubs.expect_job_spec.copy()
 233             stubs.expect_pipeline_instance = {
 234                 'name': 'submit_wf.cwl',
 235                 'state': 'RunningOnServer',
 236                 'owner_uuid': None,
 237                 "components": {
 238                     "cwl-runner": {
 239                         'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 240                         'script_parameters': {
 241                             'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 242                             'x': {"value": {
 243                                 'basename': 'blorp.txt',
 244                                 'class': 'File',
 245                                 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 246                                 "size": 16
 247                             }},
 248                             'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 249                                   'listing': [
 250                                       {
 251                                           'basename': 'renamed.txt',
 252                                           'class': 'File', 'location':
 253                                           'keep:99999999999999999999999999999998+99/file1.txt',
 254                                           'size': 0
 255                                       }
 256                                   ]}},
 257                             'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 258                             'arv:debug': True,
 259                             'arv:enable_reuse': True,
 260                             'arv:on_error': 'continue'
 261                         },
 262                         'repository': 'arvados',
 263                         'script_version': 'master',
 264                         'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 265                         'script': 'cwl-runner',
 266                         'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 267                     }
 268                 }
 269             }
 270             stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 271             stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 272             stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 273             stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 274             stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 275                 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 276                 "state": "Queued"
 277             }
 278             stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 279             stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 280
 281             cwd = os.getcwd()
 282             filepath = os.path.join(cwd, "tests/wf/submit_wf_packed.cwl")
 283             with open(filepath) as f:
 284                 expect_packed_workflow = yaml.round_trip_load(f)
 285
 286             if wfpath is None:
 287                 wfpath = wfname
 288
 289             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 290             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 291             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 292
 293             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 294             expect_packed_workflow.update(stubs.git_info)
 295
 296             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 297
 298             if wfname == wfpath:
 299                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 300             else:
 301                 container_name = wfname
 302
 303             stubs.expect_container_spec = {
 304                 'priority': 500,
 305                 'mounts': {
 306                     '/var/spool/cwl': {
 307                         'writable': True,
 308                         'kind': 'collection'
 309                     },
 310                     '/var/lib/cwl/workflow.json': {
 311                         'content': expect_packed_workflow,
 312                         'kind': 'json'
 313                     },
 314                     'stdout': {
 315                         'path': '/var/spool/cwl/cwl.output.json',
 316                         'kind': 'file'
 317                     },
 318                     '/var/lib/cwl/cwl.input.json': {
 319                         'kind': 'json',
 320                         'content': {
 321                             'y': {
 322                                 'basename': '99999999999999999999999999999998+99',
 323                                 'location': 'keep:99999999999999999999999999999998+99',
 324                                 'class': 'Directory'},
 325                             'x': {
 326                                 'basename': u'blorp.txt',
 327                                 'class': 'File',
 328                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 329                                 "size": 16
 330                             },
 331                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 332                                 {'basename': 'renamed.txt',
 333                                  'class': 'File',
 334                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 335                                  'size': 0
 336                                 }
 337                             ]}
 338                         },
 339                         'kind': 'json'
 340                     }
 341                 },
 342                 'secret_mounts': {},
 343                 'state': 'Committed',
 344                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 345                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 346                             '--eval-timeout=20', '--thread-count=0',
 347                             '--enable-reuse', "--collection-cache-size=256",
 348                             '--output-name=Output from workflow '+container_name,
 349                             '--debug', '--on-error=continue',
 350                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 351                 'name': container_name,
 352                 'container_image': '999999999999999999999999999999d3+99',
 353                 'output_name': 'Output from workflow %s' % (container_name),
 354                 'output_path': '/var/spool/cwl',
 355                 'cwd': '/var/spool/cwl',
 356                 'runtime_constraints': {
 357                     'API': True,
 358                     'vcpus': 1,
 359                     'ram': (1024+256)*1024*1024
 360                 },
 361                 'use_existing': False,
 362                 'properties': stubs.git_props,
 363                 'secret_mounts': {}
 364             }
 365
 366             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 367             stubs.api.workflows().create().execute.return_value = {
 368                 "uuid": stubs.expect_workflow_uuid,
 369             }
 370             def update_mock(**kwargs):
 371                 stubs.updated_uuid = kwargs.get('uuid')
 372                 return mock.DEFAULT
 373             stubs.api.workflows().update.side_effect = update_mock
 374             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 375                 "uuid": stubs.updated_uuid,
 376             }
 377
 378             return func(self, stubs, *args, **kwargs)
 379         return wrapped
 380     return outer_wrapper
 381
 382 class TestSubmit(unittest.TestCase):
 383
 384     def setUp(self):
 385         cwltool.process._names = set()
 386         arvados_cwl.arvdocker.arv_docker_clear_cache()
 387
 388     def tearDown(self):
 389         root_logger = logging.getLogger('')
 390
 391         # Remove existing RuntimeStatusLoggingHandlers if they exist
 392         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 393         root_logger.handlers = handlers
 394
 395     @mock.patch("time.sleep")
 396     @stubs()
 397     def test_submit_invalid_runner_ram(self, stubs, tm):
 398         exited = arvados_cwl.main(
 399             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 400              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 401             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 402         self.assertEqual(exited, 1)
 403
 404
 405     @stubs()
 406     def test_submit_container(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         stubs.api.collections().create.assert_has_calls([
 413             mock.call(body=JsonDiffMatcher({
 414                 'manifest_text':
 415                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 416                 'replication_desired': None,
 417                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 418             }), ensure_unique_name=False),
 419             mock.call(body=JsonDiffMatcher({
 420                 'manifest_text':
 421                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 422                 'replication_desired': None,
 423                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 424             }), ensure_unique_name=False),
 425             ])
 426
 427         expect_container = copy.deepcopy(stubs.expect_container_spec)
 428         stubs.api.container_requests().create.assert_called_with(
 429             body=JsonDiffMatcher(expect_container))
 430         self.assertEqual(stubs.capture_stdout.getvalue(),
 431                          stubs.expect_container_request_uuid + '\n')
 432         self.assertEqual(exited, 0)
 433
 434
 435     @stubs()
 436     def test_submit_container_tool(self, stubs):
 437         # test for issue #16139
 438         exited = arvados_cwl.main(
 439             ["--submit", "--no-wait", "--api=containers", "--debug",
 440                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 441             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 442
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_container_request_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @stubs()
 448     def test_submit_container_no_reuse(self, stubs):
 449         exited = arvados_cwl.main(
 450             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 451                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 452             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 453
 454         expect_container = copy.deepcopy(stubs.expect_container_spec)
 455         expect_container["command"] = [
 456             'arvados-cwl-runner', '--local', '--api=containers',
 457             '--no-log-timestamps', '--disable-validate', '--disable-color',
 458             '--eval-timeout=20', '--thread-count=0',
 459             '--disable-reuse', "--collection-cache-size=256",
 460             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 461             '--debug', '--on-error=continue',
 462             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 463         expect_container["use_existing"] = False
 464
 465         stubs.api.container_requests().create.assert_called_with(
 466             body=JsonDiffMatcher(expect_container))
 467         self.assertEqual(stubs.capture_stdout.getvalue(),
 468                          stubs.expect_container_request_uuid + '\n')
 469         self.assertEqual(exited, 0)
 470
 471     @stubs(('submit_wf_no_reuse.cwl', None))
 472     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 473         exited = arvados_cwl.main(
 474             ["--submit", "--no-wait", "--api=containers", "--debug",
 475              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 477         self.assertEqual(exited, 0)
 478
 479         expect_container = copy.deepcopy(stubs.expect_container_spec)
 480         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 481         expect_container["use_existing"] = False
 482         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 483             {
 484                 "class": "http://arvados.org/cwl#ReuseRequirement",
 485                 "enableReuse": False,
 486             },
 487         ]
 488         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 489             "arv": "http://arvados.org/cwl#",
 490             "cwltool": "http://commonwl.org/cwltool#"
 491         }
 492
 493         stubs.api.container_requests().create.assert_called_with(
 494             body=JsonDiffMatcher(expect_container))
 495         self.assertEqual(stubs.capture_stdout.getvalue(),
 496                          stubs.expect_container_request_uuid + '\n')
 497
 498
 499     @stubs()
 500     def test_submit_container_on_error(self, stubs):
 501         exited = arvados_cwl.main(
 502             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 503                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505
 506         expect_container = copy.deepcopy(stubs.expect_container_spec)
 507         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 508                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 509                                        '--eval-timeout=20', '--thread-count=0',
 510                                        '--enable-reuse', "--collection-cache-size=256",
 511                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 512                                        '--debug', '--on-error=stop',
 513                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(stubs.capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519         self.assertEqual(exited, 0)
 520
 521     @stubs()
 522     def test_submit_container_output_name(self, stubs):
 523         output_name = "test_output_name"
 524
 525         exited = arvados_cwl.main(
 526             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 527                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 528             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 529
 530         expect_container = copy.deepcopy(stubs.expect_container_spec)
 531         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 532                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 533                                        '--eval-timeout=20', '--thread-count=0',
 534                                        '--enable-reuse', "--collection-cache-size=256",
 535                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 536                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 537         expect_container["output_name"] = output_name
 538
 539         stubs.api.container_requests().create.assert_called_with(
 540             body=JsonDiffMatcher(expect_container))
 541         self.assertEqual(stubs.capture_stdout.getvalue(),
 542                          stubs.expect_container_request_uuid + '\n')
 543         self.assertEqual(exited, 0)
 544
 545     @stubs()
 546     def test_submit_storage_classes(self, stubs):
 547         exited = arvados_cwl.main(
 548             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 549                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 550             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 551
 552         expect_container = copy.deepcopy(stubs.expect_container_spec)
 553         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 554                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 555                                        '--eval-timeout=20', '--thread-count=0',
 556                                        '--enable-reuse', "--collection-cache-size=256",
 557                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 558                                        "--debug",
 559                                        "--storage-classes=foo", '--on-error=continue',
 560                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 561
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(stubs.capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566         self.assertEqual(exited, 0)
 567
 568     @stubs()
 569     def test_submit_multiple_storage_classes(self, stubs):
 570         exited = arvados_cwl.main(
 571             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 572                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 573             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 574
 575         expect_container = copy.deepcopy(stubs.expect_container_spec)
 576         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 577                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 578                                        '--eval-timeout=20', '--thread-count=0',
 579                                        '--enable-reuse', "--collection-cache-size=256",
 580                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 581                                        "--debug",
 582                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 583                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 584
 585         stubs.api.container_requests().create.assert_called_with(
 586             body=JsonDiffMatcher(expect_container))
 587         self.assertEqual(stubs.capture_stdout.getvalue(),
 588                          stubs.expect_container_request_uuid + '\n')
 589         self.assertEqual(exited, 0)
 590
 591     @mock.patch("cwltool.task_queue.TaskQueue")
 592     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 593     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 594     @stubs()
 595     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 596         final_output_c = arvados.collection.Collection()
 597         make_output.return_value = ({},final_output_c)
 598
 599         def set_final_output(job_order, output_callback, runtimeContext):
 600             output_callback({"out": "zzzzz"}, "success")
 601             return []
 602         job.side_effect = set_final_output
 603
 604         exited = arvados_cwl.main(
 605             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 606                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 607             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 608
 609         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 610         self.assertEqual(exited, 0)
 611
 612     @mock.patch("cwltool.task_queue.TaskQueue")
 613     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 614     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 615     @stubs()
 616     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 617         final_output_c = arvados.collection.Collection()
 618         make_output.return_value = ({},final_output_c)
 619         stubs.api.config().get.return_value = {"default": {"Default": True}}
 620
 621         def set_final_output(job_order, output_callback, runtimeContext):
 622             output_callback({"out": "zzzzz"}, "success")
 623             return []
 624         job.side_effect = set_final_output
 625
 626         exited = arvados_cwl.main(
 627             ["--debug", "--local", "--disable-git",
 628                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 629             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 630
 631         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 632         self.assertEqual(exited, 0)
 633
 634     @mock.patch("cwltool.task_queue.TaskQueue")
 635     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 636     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 637     @stubs()
 638     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 639         final_output_c = arvados.collection.Collection()
 640         make_output.return_value = ({},final_output_c)
 641
 642         def set_final_output(job_order, output_callback, runtimeContext):
 643             output_callback({"out": "zzzzz"}, "success")
 644             return []
 645         job.side_effect = set_final_output
 646
 647         exited = arvados_cwl.main(
 648             ["--debug", "--local", "--disable-git",
 649                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 650             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 651
 652         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 653         self.assertEqual(exited, 0)
 654
 655     @stubs()
 656     def test_submit_container_output_ttl(self, stubs):
 657         exited = arvados_cwl.main(
 658             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 659                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 660             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 661
 662         expect_container = copy.deepcopy(stubs.expect_container_spec)
 663         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 664                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 665                                        '--eval-timeout=20', '--thread-count=0',
 666                                        '--enable-reuse', "--collection-cache-size=256",
 667                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 668                                        '--debug',
 669                                        '--on-error=continue',
 670                                        "--intermediate-output-ttl=3600",
 671                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 672
 673         stubs.api.container_requests().create.assert_called_with(
 674             body=JsonDiffMatcher(expect_container))
 675         self.assertEqual(stubs.capture_stdout.getvalue(),
 676                          stubs.expect_container_request_uuid + '\n')
 677         self.assertEqual(exited, 0)
 678
 679     @stubs()
 680     def test_submit_container_trash_intermediate(self, stubs):
 681         exited = arvados_cwl.main(
 682             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 683                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 684             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 685
 686
 687         expect_container = copy.deepcopy(stubs.expect_container_spec)
 688         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 689                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 690                                        '--eval-timeout=20', '--thread-count=0',
 691                                        '--enable-reuse', "--collection-cache-size=256",
 692                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 693                                        '--debug', '--on-error=continue',
 694                                        "--trash-intermediate",
 695                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 696
 697         stubs.api.container_requests().create.assert_called_with(
 698             body=JsonDiffMatcher(expect_container))
 699         self.assertEqual(stubs.capture_stdout.getvalue(),
 700                          stubs.expect_container_request_uuid + '\n')
 701         self.assertEqual(exited, 0)
 702
 703     @stubs()
 704     def test_submit_container_output_tags(self, stubs):
 705         output_tags = "tag0,tag1,tag2"
 706
 707         exited = arvados_cwl.main(
 708             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 709                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 710             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 711
 712         expect_container = copy.deepcopy(stubs.expect_container_spec)
 713         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 714                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 715                                        '--eval-timeout=20', '--thread-count=0',
 716                                        '--enable-reuse', "--collection-cache-size=256",
 717                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 718                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 719                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 720
 721         stubs.api.container_requests().create.assert_called_with(
 722             body=JsonDiffMatcher(expect_container))
 723         self.assertEqual(stubs.capture_stdout.getvalue(),
 724                          stubs.expect_container_request_uuid + '\n')
 725         self.assertEqual(exited, 0)
 726
 727     @stubs()
 728     def test_submit_container_runner_ram(self, stubs):
 729         exited = arvados_cwl.main(
 730             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 731                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 732             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 733
 734         expect_container = copy.deepcopy(stubs.expect_container_spec)
 735         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 736
 737         stubs.api.container_requests().create.assert_called_with(
 738             body=JsonDiffMatcher(expect_container))
 739         self.assertEqual(stubs.capture_stdout.getvalue(),
 740                          stubs.expect_container_request_uuid + '\n')
 741         self.assertEqual(exited, 0)
 742
 743     @mock.patch("arvados.collection.CollectionReader")
 744     @mock.patch("time.sleep")
 745     @stubs()
 746     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 747         collectionReader().exists.return_value = True
 748         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 749         exited = arvados_cwl.main(
 750             ["--submit", "--no-wait", "--api=containers", "--debug",
 751              "tests/wf/submit_keepref_wf.cwl"],
 752             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 753         self.assertEqual(exited, 0)
 754
 755     @mock.patch("arvados.collection.CollectionReader")
 756     @mock.patch("time.sleep")
 757     @stubs()
 758     def test_submit_keepref(self, stubs, tm, reader):
 759         with open("tests/wf/expect_arvworkflow.cwl") as f:
 760             reader().open().__enter__().read.return_value = f.read()
 761
 762         exited = arvados_cwl.main(
 763             ["--submit", "--no-wait", "--api=containers", "--debug",
 764              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 765             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 766
 767         expect_container = {
 768             'priority': 500,
 769             'mounts': {
 770                 '/var/spool/cwl': {
 771                     'writable': True,
 772                     'kind': 'collection'
 773                 },
 774                 'stdout': {
 775                     'path': '/var/spool/cwl/cwl.output.json',
 776                     'kind': 'file'
 777                 },
 778                 '/var/lib/cwl/workflow': {
 779                     'portable_data_hash': '99999999999999999999999999999994+99',
 780                     'kind': 'collection'
 781                 },
 782                 '/var/lib/cwl/cwl.input.json': {
 783                     'content': {
 784                         'x': 'XxX'
 785                     },
 786                     'kind': 'json'
 787                 }
 788             }, 'state': 'Committed',
 789             'output_path': '/var/spool/cwl',
 790             'name': 'expect_arvworkflow.cwl#main',
 791             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 792             'container_image': '999999999999999999999999999999d3+99',
 793             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 794                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 795                         '--eval-timeout=20', '--thread-count=0',
 796                         '--enable-reuse', "--collection-cache-size=256",
 797                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 798                         '--debug', '--on-error=continue',
 799                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 800             'cwd': '/var/spool/cwl',
 801             'runtime_constraints': {
 802                 'API': True,
 803                 'vcpus': 1,
 804                 'ram': 1342177280
 805             },
 806             'use_existing': False,
 807             'properties': {},
 808             'secret_mounts': {}
 809         }
 810
 811         stubs.api.container_requests().create.assert_called_with(
 812             body=JsonDiffMatcher(expect_container))
 813         self.assertEqual(stubs.capture_stdout.getvalue(),
 814                          stubs.expect_container_request_uuid + '\n')
 815         self.assertEqual(exited, 0)
 816
 817     @mock.patch("time.sleep")
 818     @stubs()
 819     def test_submit_arvworkflow(self, stubs, tm):
 820         with open("tests/wf/expect_arvworkflow.cwl") as f:
 821             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 822
 823         exited = arvados_cwl.main(
 824             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 825              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 826             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 827
 828         expect_container = {
 829             'priority': 500,
 830             'mounts': {
 831                 '/var/spool/cwl': {
 832                     'writable': True,
 833                     'kind': 'collection'
 834                 },
 835                 'stdout': {
 836                     'path': '/var/spool/cwl/cwl.output.json',
 837                     'kind': 'file'
 838                 },
 839                 '/var/lib/cwl/workflow.json': {
 840                     'kind': 'json',
 841                     'content': {
 842                         'cwlVersion': 'v1.0',
 843                         '$graph': [
 844                             {
 845                                 'id': '#main',
 846                                 'inputs': [
 847                                     {'type': 'string', 'id': '#main/x'}
 848                                 ],
 849                                 'steps': [
 850                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 851                                      'run': '#submit_tool.cwl',
 852                                      'id': '#main/step1',
 853                                      'out': []}
 854                                 ],
 855                                 'class': 'Workflow',
 856                                 'outputs': []
 857                             },
 858                             {
 859                                 'inputs': [
 860                                     {
 861                                         'inputBinding': {'position': 1},
 862                                         'type': 'string',
 863                                         'id': '#submit_tool.cwl/x'}
 864                                 ],
 865                                 'requirements': [
 866                                     {
 867                                         'dockerPull': 'debian:buster-slim',
 868                                         'class': 'DockerRequirement'
 869                                     }
 870                                 ],
 871                                 'id': '#submit_tool.cwl',
 872                                 'outputs': [],
 873                                 'baseCommand': 'cat',
 874                                 'class': 'CommandLineTool'
 875                             }
 876                         ]
 877                     }
 878                 },
 879                 '/var/lib/cwl/cwl.input.json': {
 880                     'content': {
 881                         'x': 'XxX'
 882                     },
 883                     'kind': 'json'
 884                 }
 885             }, 'state': 'Committed',
 886             'output_path': '/var/spool/cwl',
 887             'name': 'a test workflow',
 888             'container_image': "999999999999999999999999999999d3+99",
 889             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 890                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 891                         '--eval-timeout=20', '--thread-count=0',
 892                         '--enable-reuse', "--collection-cache-size=256",
 893                         "--output-name=Output from workflow a test workflow",
 894                         '--debug', '--on-error=continue',
 895                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 896             'output_name': 'Output from workflow a test workflow',
 897             'cwd': '/var/spool/cwl',
 898             'runtime_constraints': {
 899                 'API': True,
 900                 'vcpus': 1,
 901                 'ram': 1342177280
 902             },
 903             'use_existing': False,
 904             'properties': {
 905                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 906             },
 907             'secret_mounts': {}
 908         }
 909
 910         stubs.api.container_requests().create.assert_called_with(
 911             body=JsonDiffMatcher(expect_container))
 912         self.assertEqual(stubs.capture_stdout.getvalue(),
 913                          stubs.expect_container_request_uuid + '\n')
 914         self.assertEqual(exited, 0)
 915
 916     @stubs(('hello container 123', 'submit_wf.cwl'))
 917     def test_submit_container_name(self, stubs):
 918         exited = arvados_cwl.main(
 919             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 920                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 921             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 922
 923         expect_container = copy.deepcopy(stubs.expect_container_spec)
 924
 925         stubs.api.container_requests().create.assert_called_with(
 926             body=JsonDiffMatcher(expect_container))
 927         self.assertEqual(stubs.capture_stdout.getvalue(),
 928                          stubs.expect_container_request_uuid + '\n')
 929         self.assertEqual(exited, 0)
 930
 931     @stubs()
 932     def test_submit_missing_input(self, stubs):
 933         exited = arvados_cwl.main(
 934             ["--submit", "--no-wait", "--api=containers", "--debug",
 935              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 936             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 937         self.assertEqual(exited, 0)
 938
 939         exited = arvados_cwl.main(
 940             ["--submit", "--no-wait", "--api=containers", "--debug",
 941              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 942             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 943         self.assertEqual(exited, 1)
 944
 945     @stubs()
 946     def test_submit_container_project(self, stubs):
 947         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 948         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 949         exited = arvados_cwl.main(
 950             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 951                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 952             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 953
 954         expect_container = copy.deepcopy(stubs.expect_container_spec)
 955         expect_container["owner_uuid"] = project_uuid
 956         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 957                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 958                                        "--eval-timeout=20", "--thread-count=0",
 959                                        '--enable-reuse', "--collection-cache-size=256",
 960                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 961                                        '--debug',
 962                                        '--on-error=continue',
 963                                        '--project-uuid='+project_uuid,
 964                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 965
 966         stubs.api.container_requests().create.assert_called_with(
 967             body=JsonDiffMatcher(expect_container))
 968         self.assertEqual(stubs.capture_stdout.getvalue(),
 969                          stubs.expect_container_request_uuid + '\n')
 970         self.assertEqual(exited, 0)
 971
 972     @stubs()
 973     def test_submit_container_eval_timeout(self, stubs):
 974         exited = arvados_cwl.main(
 975             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 976                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 977             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 978
 979         expect_container = copy.deepcopy(stubs.expect_container_spec)
 980         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 981                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 982                                        '--eval-timeout=60.0', '--thread-count=0',
 983                                        '--enable-reuse', "--collection-cache-size=256",
 984                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 985                                        '--debug', '--on-error=continue',
 986                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 987
 988         stubs.api.container_requests().create.assert_called_with(
 989             body=JsonDiffMatcher(expect_container))
 990         self.assertEqual(stubs.capture_stdout.getvalue(),
 991                          stubs.expect_container_request_uuid + '\n')
 992         self.assertEqual(exited, 0)
 993
 994     @stubs()
 995     def test_submit_container_collection_cache(self, stubs):
 996         exited = arvados_cwl.main(
 997             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 998                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 999             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1000
1001         expect_container = copy.deepcopy(stubs.expect_container_spec)
1002         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1003                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1004                                        '--eval-timeout=20', '--thread-count=0',
1005                                        '--enable-reuse', "--collection-cache-size=500",
1006                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1007                                        '--debug', '--on-error=continue',
1008                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1009         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1010
1011         stubs.api.container_requests().create.assert_called_with(
1012             body=JsonDiffMatcher(expect_container))
1013         self.assertEqual(stubs.capture_stdout.getvalue(),
1014                          stubs.expect_container_request_uuid + '\n')
1015         self.assertEqual(exited, 0)
1016
1017     @stubs()
1018     def test_submit_container_thread_count(self, stubs):
1019         exited = arvados_cwl.main(
1020             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1021                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1022             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1023
1024         expect_container = copy.deepcopy(stubs.expect_container_spec)
1025         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1026                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1027                                        '--eval-timeout=20', '--thread-count=20',
1028                                        '--enable-reuse', "--collection-cache-size=256",
1029                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1030                                        '--debug', '--on-error=continue',
1031                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1032
1033         stubs.api.container_requests().create.assert_called_with(
1034             body=JsonDiffMatcher(expect_container))
1035         self.assertEqual(stubs.capture_stdout.getvalue(),
1036                          stubs.expect_container_request_uuid + '\n')
1037         self.assertEqual(exited, 0)
1038
1039     @stubs()
1040     def test_submit_container_runner_image(self, stubs):
1041         exited = arvados_cwl.main(
1042             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1043                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1044             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1045
1046         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1047
1048         expect_container = copy.deepcopy(stubs.expect_container_spec)
1049         stubs.api.container_requests().create.assert_called_with(
1050             body=JsonDiffMatcher(expect_container))
1051         self.assertEqual(stubs.capture_stdout.getvalue(),
1052                          stubs.expect_container_request_uuid + '\n')
1053         self.assertEqual(exited, 0)
1054
1055     @stubs()
1056     def test_submit_priority(self, stubs):
1057         exited = arvados_cwl.main(
1058             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1059                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1060             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1061
1062         stubs.expect_container_spec["priority"] = 669
1063
1064         expect_container = copy.deepcopy(stubs.expect_container_spec)
1065         stubs.api.container_requests().create.assert_called_with(
1066             body=JsonDiffMatcher(expect_container))
1067         self.assertEqual(stubs.capture_stdout.getvalue(),
1068                          stubs.expect_container_request_uuid + '\n')
1069         self.assertEqual(exited, 0)
1070
1071     @stubs(('submit_wf_runner_resources.cwl', None))
1072     def test_submit_wf_runner_resources(self, stubs):
1073         exited = arvados_cwl.main(
1074             ["--submit", "--no-wait", "--api=containers", "--debug",
1075                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1076             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1077
1078         expect_container = copy.deepcopy(stubs.expect_container_spec)
1079         expect_container["runtime_constraints"] = {
1080             "API": True,
1081             "vcpus": 2,
1082             "ram": (2000+512) * 2**20
1083         }
1084         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1085             {
1086                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1087                 "coresMin": 2,
1088                 "ramMin": 2000,
1089                 "keep_cache": 512
1090             }
1091         ]
1092         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1093             "arv": "http://arvados.org/cwl#",
1094         }
1095         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1096
1097         stubs.api.container_requests().create.assert_called_with(
1098             body=JsonDiffMatcher(expect_container))
1099         self.assertEqual(stubs.capture_stdout.getvalue(),
1100                          stubs.expect_container_request_uuid + '\n')
1101         self.assertEqual(exited, 0)
1102
1103     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1104     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1105     @mock.patch("arvados.api")
1106     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1107         arvados_cwl.arvdocker.arv_docker_clear_cache()
1108
1109         arvrunner = mock.MagicMock()
1110         arvrunner.project_uuid = ""
1111         api.return_value = mock.MagicMock()
1112         arvrunner.api = api.return_value
1113         arvrunner.runtimeContext.match_local_docker = False
1114         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1115                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1116                                                                         "link_class": "docker_image_repo+tag",
1117                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1118                                                                         "owner_uuid": "",
1119                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1120                                                             {"items": [{"created_at": "",
1121                                                                         "head_uuid": "",
1122                                                                         "link_class": "docker_image_hash",
1123                                                                         "name": "123456",
1124                                                                         "owner_uuid": "",
1125                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1126                                                             {"items": [{"created_at": "",
1127                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1128                                                                         "link_class": "docker_image_repo+tag",
1129                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1130                                                                         "owner_uuid": "",
1131                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1132                                                             {"items": [{"created_at": "",
1133                                                                         "head_uuid": "",
1134                                                                         "link_class": "docker_image_hash",
1135                                                                         "name": "123456",
1136                                                                         "owner_uuid": "",
1137                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1138         )
1139         find_one_image_hash.return_value = "123456"
1140
1141         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1142                                                                               "owner_uuid": "",
1143                                                                               "manifest_text": "",
1144                                                                               "properties": ""
1145                                                                               }], "items_available": 1, "offset": 0},
1146                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1147                                                                               "owner_uuid": "",
1148                                                                               "manifest_text": "",
1149                                                                               "properties": ""
1150                                                                           }], "items_available": 1, "offset": 0})
1151         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1152         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1153                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1154
1155         self.assertEqual("9999999999999999999999999999999b+99",
1156                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1157
1158
1159     @stubs()
1160     def test_submit_secrets(self, stubs):
1161         exited = arvados_cwl.main(
1162             ["--submit", "--no-wait", "--api=containers", "--debug",
1163                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1164             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1165
1166         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1167         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1168
1169         expect_container = {
1170             "command": [
1171                 "arvados-cwl-runner",
1172                 "--local",
1173                 "--api=containers",
1174                 "--no-log-timestamps",
1175                 "--disable-validate",
1176                 "--disable-color",
1177                 "--eval-timeout=20",
1178                 '--thread-count=0',
1179                 "--enable-reuse",
1180                 "--collection-cache-size=256",
1181                 '--output-name=Output from workflow secret_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1182                 "--debug",
1183                 "--on-error=continue",
1184                 "/var/lib/cwl/workflow.json#main",
1185                 "/var/lib/cwl/cwl.input.json"
1186             ],
1187             "container_image": "999999999999999999999999999999d3+99",
1188             "cwd": "/var/spool/cwl",
1189             "mounts": {
1190                 "/var/lib/cwl/cwl.input.json": {
1191                     "content": {
1192                         "pw": {
1193                             "$include": "/secrets/s0"
1194                         }
1195                     },
1196                     "kind": "json"
1197                 },
1198                 "/var/lib/cwl/workflow.json": {
1199                     "content": {
1200                         "$graph": [
1201                             {
1202                                 "arguments": [
1203                                     "md5sum",
1204                                     "example.conf"
1205                                 ],
1206                                 "class": "CommandLineTool",
1207                                 "hints": [
1208                                     {
1209                                         "class": "http://commonwl.org/cwltool#Secrets",
1210                                         "secrets": [
1211                                             "#secret_job.cwl/pw"
1212                                         ]
1213                                     }
1214                                 ],
1215                                 "id": "#secret_job.cwl",
1216                                 "inputs": [
1217                                     {
1218                                         "id": "#secret_job.cwl/pw",
1219                                         "type": "string"
1220                                     }
1221                                 ],
1222                                 "outputs": [
1223                                     {
1224                                         "id": "#secret_job.cwl/out",
1225                                         "type": "File",
1226                                         "outputBinding": {
1227                                               "glob": "hashed_example.txt"
1228                                         }
1229                                     }
1230                                 ],
1231                                 "stdout": "hashed_example.txt",
1232                                 "requirements": [
1233                                     {
1234                                         "class": "InitialWorkDirRequirement",
1235                                         "listing": [
1236                                             {
1237                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1238                                                 "entryname": "example.conf"
1239                                             }
1240                                         ]
1241                                     }
1242                                 ]
1243                             },
1244                             {
1245                                 "class": "Workflow",
1246                                 "hints": [
1247                                     {
1248                                         "class": "DockerRequirement",
1249                                         "dockerPull": "debian:buster-slim",
1250                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1251                                     },
1252                                     {
1253                                         "class": "http://commonwl.org/cwltool#Secrets",
1254                                         "secrets": [
1255                                             "#main/pw"
1256                                         ]
1257                                     }
1258                                 ],
1259                                 "id": "#main",
1260                                 "inputs": [
1261                                     {
1262                                         "id": "#main/pw",
1263                                         "type": "string"
1264                                     }
1265                                 ],
1266                                 "outputs": [
1267                                     {
1268                                         "id": "#main/out",
1269                                         "outputSource": "#main/step1/out",
1270                                         "type": "File"
1271                                     }
1272                                 ],
1273                                 "steps": [
1274                                     {
1275                                         "id": "#main/step1",
1276                                         "in": [
1277                                             {
1278                                                 "id": "#main/step1/pw",
1279                                                 "source": "#main/pw"
1280                                             }
1281                                         ],
1282                                         "out": [
1283                                             "#main/step1/out"
1284                                         ],
1285                                         "run": "#secret_job.cwl"
1286                                     }
1287                                 ]
1288                             }
1289                         ],
1290                         "$namespaces": {
1291                             "cwltool": "http://commonwl.org/cwltool#"
1292                         },
1293                         "cwlVersion": "v1.0"
1294                     },
1295                     "kind": "json"
1296                 },
1297                 "/var/spool/cwl": {
1298                     "kind": "collection",
1299                     "writable": True
1300                 },
1301                 "stdout": {
1302                     "kind": "file",
1303                     "path": "/var/spool/cwl/cwl.output.json"
1304                 }
1305             },
1306             "name": "secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1307             "output_name": "Output from workflow secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1308             "output_path": "/var/spool/cwl",
1309             "priority": 500,
1310             "properties": stubs.git_props,
1311             "runtime_constraints": {
1312                 "API": True,
1313                 "ram": 1342177280,
1314                 "vcpus": 1
1315             },
1316             "secret_mounts": {
1317                 "/secrets/s0": {
1318                     "content": "blorp",
1319                     "kind": "text"
1320                 }
1321             },
1322             "state": "Committed",
1323             "use_existing": False
1324         }
1325
1326         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"].update(stubs.git_info)
1327
1328         stubs.api.container_requests().create.assert_called_with(
1329             body=JsonDiffMatcher(expect_container))
1330         self.assertEqual(stubs.capture_stdout.getvalue(),
1331                          stubs.expect_container_request_uuid + '\n')
1332         self.assertEqual(exited, 0)
1333
1334     @stubs()
1335     def test_submit_request_uuid(self, stubs):
1336         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1337         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1338
1339         stubs.api.container_requests().update().execute.return_value = {
1340             "uuid": stubs.expect_container_request_uuid,
1341             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1342             "state": "Queued"
1343         }
1344
1345         exited = arvados_cwl.main(
1346             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1347                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1348             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1349
1350         stubs.api.container_requests().update.assert_called_with(
1351             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1352         self.assertEqual(stubs.capture_stdout.getvalue(),
1353                          stubs.expect_container_request_uuid + '\n')
1354         self.assertEqual(exited, 0)
1355
1356     @stubs()
1357     def test_submit_container_cluster_id(self, stubs):
1358         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1359
1360         exited = arvados_cwl.main(
1361             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1362                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1363             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1364
1365         expect_container = copy.deepcopy(stubs.expect_container_spec)
1366
1367         stubs.api.container_requests().create.assert_called_with(
1368             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1369         self.assertEqual(stubs.capture_stdout.getvalue(),
1370                          stubs.expect_container_request_uuid + '\n')
1371         self.assertEqual(exited, 0)
1372
1373     @stubs()
1374     def test_submit_validate_cluster_id(self, stubs):
1375         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1376         exited = arvados_cwl.main(
1377             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1378              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1379             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1380         self.assertEqual(exited, 1)
1381
1382     @stubs()
1383     def test_submit_validate_project_uuid(self, stubs):
1384         # Fails with bad cluster prefix
1385         exited = arvados_cwl.main(
1386             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1388             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1389         self.assertEqual(exited, 1)
1390
1391         # Project lookup fails
1392         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1393         exited = arvados_cwl.main(
1394             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1395              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1396             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1397         self.assertEqual(exited, 1)
1398
1399         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1400         exited = arvados_cwl.main(
1401             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1402              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1403             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1404         self.assertEqual(exited, 0)
1405
1406
1407     @mock.patch("arvados.collection.CollectionReader")
1408     @stubs()
1409     def test_submit_uuid_inputs(self, stubs, collectionReader):
1410         collectionReader().exists.return_value = True
1411         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1412         def list_side_effect(**kwargs):
1413             m = mock.MagicMock()
1414             if "count" in kwargs:
1415                 m.execute.return_value = {"items": [
1416                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1417                 ]}
1418             else:
1419                 m.execute.return_value = {"items": []}
1420             return m
1421         stubs.api.collections().list.side_effect = list_side_effect
1422
1423         exited = arvados_cwl.main(
1424             ["--submit", "--no-wait", "--api=containers", "--debug",
1425                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1426             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1427
1428         expect_container = copy.deepcopy(stubs.expect_container_spec)
1429         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1430         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1431         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1432
1433         stubs.api.collections().list.assert_has_calls([
1434             mock.call(count='none',
1435                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1436                       select=['uuid', 'portable_data_hash'])])
1437         stubs.api.container_requests().create.assert_called_with(
1438             body=JsonDiffMatcher(expect_container))
1439         self.assertEqual(stubs.capture_stdout.getvalue(),
1440                          stubs.expect_container_request_uuid + '\n')
1441         self.assertEqual(exited, 0)
1442
1443     @stubs()
1444     def test_submit_mismatched_uuid_inputs(self, stubs):
1445         def list_side_effect(**kwargs):
1446             m = mock.MagicMock()
1447             if "count" in kwargs:
1448                 m.execute.return_value = {"items": [
1449                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1450                 ]}
1451             else:
1452                 m.execute.return_value = {"items": []}
1453             return m
1454         stubs.api.collections().list.side_effect = list_side_effect
1455
1456         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1457             capture_stderr = StringIO()
1458             cwltool_logger = logging.getLogger('cwltool')
1459             stderr_logger = logging.StreamHandler(capture_stderr)
1460             cwltool_logger.addHandler(stderr_logger)
1461
1462             try:
1463                 exited = arvados_cwl.main(
1464                     ["--submit", "--no-wait", "--api=containers", "--debug",
1465                         "tests/wf/submit_wf.cwl", infile],
1466                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1467
1468                 self.assertEqual(exited, 1)
1469                 self.assertRegex(
1470                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1471                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1472             finally:
1473                 cwltool_logger.removeHandler(stderr_logger)
1474
1475     @mock.patch("arvados.collection.CollectionReader")
1476     @stubs()
1477     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1478         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1479         capture_stderr = StringIO()
1480
1481         cwltool_logger = logging.getLogger('cwltool')
1482         stderr_logger = logging.StreamHandler(capture_stderr)
1483         cwltool_logger.addHandler(stderr_logger)
1484
1485         exited = arvados_cwl.main(
1486             ["--submit", "--no-wait", "--api=containers", "--debug",
1487                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1488             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1489
1490         try:
1491             self.assertEqual(exited, 1)
1492             self.assertRegex(
1493                 capture_stderr.getvalue(),
1494                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1495         finally:
1496             cwltool_logger.removeHandler(stderr_logger)
1497
1498     @stubs(('submit_wf_process_properties.cwl', None))
1499     def test_submit_set_process_properties(self, stubs):
1500         exited = arvados_cwl.main(
1501             ["--submit", "--no-wait", "--api=containers", "--debug",
1502                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1503             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1504
1505         expect_container = copy.deepcopy(stubs.expect_container_spec)
1506
1507         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1508             {
1509                 "class": "http://arvados.org/cwl#ProcessProperties",
1510                 "processProperties": [
1511                     {"propertyName": "baz",
1512                      "propertyValue": "$(inputs.x.basename)"},
1513                     {"propertyName": "foo",
1514                      "propertyValue": "bar"},
1515                     {"propertyName": "quux",
1516                      "propertyValue": {
1517                          "q1": 1,
1518                          "q2": 2
1519                      }
1520                     }
1521                 ],
1522             }
1523         ]
1524         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1525             "arv": "http://arvados.org/cwl#"
1526         }
1527
1528         expect_container["properties"].update({
1529             "baz": "blorp.txt",
1530             "foo": "bar",
1531             "quux": {
1532                 "q1": 1,
1533                 "q2": 2
1534             }
1535         })
1536
1537         stubs.api.container_requests().create.assert_called_with(
1538             body=JsonDiffMatcher(expect_container))
1539         self.assertEqual(stubs.capture_stdout.getvalue(),
1540                          stubs.expect_container_request_uuid + '\n')
1541         self.assertEqual(exited, 0)
1542
1543
1544     @stubs()
1545     def test_submit_enable_preemptible(self, stubs):
1546         exited = arvados_cwl.main(
1547             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1548                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1549             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1550
1551         expect_container = copy.deepcopy(stubs.expect_container_spec)
1552         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1553                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1554                                        '--eval-timeout=20', '--thread-count=0',
1555                                        '--enable-reuse', "--collection-cache-size=256",
1556                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1557                                        '--debug', '--on-error=continue',
1558                                        '--enable-preemptible',
1559                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1560
1561         stubs.api.container_requests().create.assert_called_with(
1562             body=JsonDiffMatcher(expect_container))
1563         self.assertEqual(stubs.capture_stdout.getvalue(),
1564                          stubs.expect_container_request_uuid + '\n')
1565         self.assertEqual(exited, 0)
1566
1567     @stubs()
1568     def test_submit_disable_preemptible(self, stubs):
1569         exited = arvados_cwl.main(
1570             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1571                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1572             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1573
1574         expect_container = copy.deepcopy(stubs.expect_container_spec)
1575         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1576                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1577                                        '--eval-timeout=20', '--thread-count=0',
1578                                        '--enable-reuse', "--collection-cache-size=256",
1579                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1580                                        '--debug', '--on-error=continue',
1581                                        '--disable-preemptible',
1582                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1583
1584         stubs.api.container_requests().create.assert_called_with(
1585             body=JsonDiffMatcher(expect_container))
1586         self.assertEqual(stubs.capture_stdout.getvalue(),
1587                          stubs.expect_container_request_uuid + '\n')
1588         self.assertEqual(exited, 0)
1589
1590
1591 class TestCreateWorkflow(unittest.TestCase):
1592     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1593     expect_workflow = StripYAMLComments(
1594         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1595
1596     def setUp(self):
1597         cwltool.process._names = set()
1598         arvados_cwl.arvdocker.arv_docker_clear_cache()
1599
1600     def tearDown(self):
1601         root_logger = logging.getLogger('')
1602
1603         # Remove existing RuntimeStatusLoggingHandlers if they exist
1604         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1605         root_logger.handlers = handlers
1606
1607     @stubs()
1608     def test_create(self, stubs):
1609         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1610         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1611
1612         exited = arvados_cwl.main(
1613             ["--create-workflow", "--debug",
1614              "--api=containers",
1615              "--project-uuid", project_uuid,
1616              "--disable-git",
1617              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1618             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1619
1620         stubs.api.pipeline_templates().create.refute_called()
1621         stubs.api.container_requests().create.refute_called()
1622
1623         body = {
1624             "workflow": {
1625                 "owner_uuid": project_uuid,
1626                 "name": "submit_wf.cwl",
1627                 "description": "",
1628                 "definition": self.expect_workflow,
1629             }
1630         }
1631         stubs.api.workflows().create.assert_called_with(
1632             body=JsonDiffMatcher(body))
1633
1634         self.assertEqual(stubs.capture_stdout.getvalue(),
1635                          stubs.expect_workflow_uuid + '\n')
1636         self.assertEqual(exited, 0)
1637
1638     @stubs()
1639     def test_create_name(self, stubs):
1640         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1641         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1642
1643         exited = arvados_cwl.main(
1644             ["--create-workflow", "--debug",
1645              "--api=containers",
1646              "--project-uuid", project_uuid,
1647              "--name", "testing 123",
1648              "--disable-git",
1649              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1650             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1651
1652         stubs.api.pipeline_templates().create.refute_called()
1653         stubs.api.container_requests().create.refute_called()
1654
1655         body = {
1656             "workflow": {
1657                 "owner_uuid": project_uuid,
1658                 "name": "testing 123",
1659                 "description": "",
1660                 "definition": self.expect_workflow,
1661             }
1662         }
1663         stubs.api.workflows().create.assert_called_with(
1664             body=JsonDiffMatcher(body))
1665
1666         self.assertEqual(stubs.capture_stdout.getvalue(),
1667                          stubs.expect_workflow_uuid + '\n')
1668         self.assertEqual(exited, 0)
1669
1670
1671     @stubs()
1672     def test_update(self, stubs):
1673         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1674         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1675
1676         exited = arvados_cwl.main(
1677             ["--update-workflow", self.existing_workflow_uuid,
1678              "--debug",
1679              "--disable-git",
1680              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1681             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1682
1683         body = {
1684             "workflow": {
1685                 "name": "submit_wf.cwl",
1686                 "description": "",
1687                 "definition": self.expect_workflow,
1688                 "owner_uuid": project_uuid
1689             }
1690         }
1691         stubs.api.workflows().update.assert_called_with(
1692             uuid=self.existing_workflow_uuid,
1693             body=JsonDiffMatcher(body))
1694         self.assertEqual(stubs.capture_stdout.getvalue(),
1695                          self.existing_workflow_uuid + '\n')
1696         self.assertEqual(exited, 0)
1697
1698
1699     @stubs()
1700     def test_update_name(self, stubs):
1701         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1702         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1703
1704         exited = arvados_cwl.main(
1705             ["--update-workflow", self.existing_workflow_uuid,
1706              "--debug", "--name", "testing 123",
1707              "--disable-git",
1708              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1709             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1710
1711         body = {
1712             "workflow": {
1713                 "name": "testing 123",
1714                 "description": "",
1715                 "definition": self.expect_workflow,
1716                 "owner_uuid": project_uuid
1717             }
1718         }
1719         stubs.api.workflows().update.assert_called_with(
1720             uuid=self.existing_workflow_uuid,
1721             body=JsonDiffMatcher(body))
1722         self.assertEqual(stubs.capture_stdout.getvalue(),
1723                          self.existing_workflow_uuid + '\n')
1724         self.assertEqual(exited, 0)
1725
1726     @stubs()
1727     def test_create_collection_per_tool(self, stubs):
1728         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1729         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1730
1731         exited = arvados_cwl.main(
1732             ["--create-workflow", "--debug",
1733              "--api=containers",
1734              "--project-uuid", project_uuid,
1735              "--disable-git",
1736              "tests/collection_per_tool/collection_per_tool.cwl"],
1737             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1738
1739         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1740         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1741
1742         body = {
1743             "workflow": {
1744                 "owner_uuid": project_uuid,
1745                 "name": "collection_per_tool.cwl",
1746                 "description": "",
1747                 "definition": expect_workflow,
1748             }
1749         }
1750         stubs.api.workflows().create.assert_called_with(
1751             body=JsonDiffMatcher(body))
1752
1753         self.assertEqual(stubs.capture_stdout.getvalue(),
1754                          stubs.expect_workflow_uuid + '\n')
1755         self.assertEqual(exited, 0)
1756
1757     @stubs()
1758     def test_create_with_imports(self, stubs):
1759         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1760         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1761
1762         exited = arvados_cwl.main(
1763             ["--create-workflow", "--debug",
1764              "--api=containers",
1765              "--project-uuid", project_uuid,
1766              "tests/wf/feddemo/feddemo.cwl"],
1767             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1768
1769         stubs.api.pipeline_templates().create.refute_called()
1770         stubs.api.container_requests().create.refute_called()
1771
1772         self.assertEqual(stubs.capture_stdout.getvalue(),
1773                          stubs.expect_workflow_uuid + '\n')
1774         self.assertEqual(exited, 0)
1775
1776     @stubs()
1777     def test_create_with_no_input(self, stubs):
1778         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1779         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1780
1781         exited = arvados_cwl.main(
1782             ["--create-workflow", "--debug",
1783              "--api=containers",
1784              "--project-uuid", project_uuid,
1785              "tests/wf/revsort/revsort.cwl"],
1786             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1787
1788         stubs.api.pipeline_templates().create.refute_called()
1789         stubs.api.container_requests().create.refute_called()
1790
1791         self.assertEqual(stubs.capture_stdout.getvalue(),
1792                          stubs.expect_workflow_uuid + '\n')
1793         self.assertEqual(exited, 0)