sdk/cwl/tests/test_submit.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 from future import standard_library
   6 standard_library.install_aliases()
   7 from builtins import object
   8 from builtins import str
   9 from future.utils import viewvalues
  10
  11 import copy
  12 import io
  13 import functools
  14 import hashlib
  15 import json
  16 import logging
  17 import mock
  18 import sys
  19 import unittest
  20 import cwltool.process
  21 import re
  22 import os
  23
  24 from io import BytesIO
  25
  26 # StringIO.StringIO and io.StringIO have different behavior write() is
  27 # called with both python2 (byte) strings and unicode strings
  28 # (specifically there's some logging in cwltool that causes trouble).
  29 # This isn't a problem on python3 because all string are unicode.
  30 if sys.version_info[0] < 3:
  31     from StringIO import StringIO
  32 else:
  33     from io import StringIO
  34
  35 import arvados
  36 import arvados.collection
  37 import arvados_cwl
  38 import arvados_cwl.executor
  39 import arvados_cwl.runner
  40 import arvados.keep
  41
  42 from .matcher import JsonDiffMatcher, StripYAMLComments
  43 from .mock_discovery import get_rootDesc
  44
  45 import ruamel.yaml as yaml
  46
  47 _rootDesc = None
  48
  49 def stubs(wfdetails=('submit_wf.cwl', None)):
  50     def outer_wrapper(func, *rest):
  51         @functools.wraps(func)
  52         @mock.patch("arvados_cwl.arvdocker.determine_image_id")
  53         @mock.patch("uuid.uuid4")
  54         @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  55         @mock.patch("arvados.collection.KeepClient")
  56         @mock.patch("arvados.keep.KeepClient")
  57         @mock.patch("arvados.events.subscribe")
  58         def wrapped(self, events, keep_client1, keep_client2, keepdocker,
  59                     uuid4, determine_image_id, *args, **kwargs):
  60             class Stubs(object):
  61                 pass
  62
  63             wfname = wfdetails[0]
  64             wfpath = wfdetails[1]
  65
  66             stubs = Stubs()
  67             stubs.events = events
  68             stubs.keepdocker = keepdocker
  69
  70             uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
  71                                  "df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
  72                                  "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
  73
  74             determine_image_id.return_value = None
  75
  76             def putstub(p, **kwargs):
  77                 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  78             keep_client1().put.side_effect = putstub
  79             keep_client1.put.side_effect = putstub
  80             keep_client2().put.side_effect = putstub
  81             keep_client2.put.side_effect = putstub
  82
  83             stubs.keep_client = keep_client2
  84             stubs.docker_images = {
  85                 "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
  86                 "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
  87                 "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
  88                 "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
  89             }
  90             def kd(a, b, image_name=None, image_tag=None, project_uuid=None):
  91                 return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
  92             stubs.keepdocker.side_effect = kd
  93
  94             stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  95             stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
  96
  97             if sys.version_info[0] < 3:
  98                 stubs.capture_stdout = BytesIO()
  99             else:
 100                 stubs.capture_stdout = StringIO()
 101
 102             stubs.api = mock.MagicMock()
 103             stubs.api._rootDesc = get_rootDesc()
 104             stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
 105             stubs.api._rootDesc["revision"] = "20210628"
 106
 107             stubs.api.users().current().execute.return_value = {
 108                 "uuid": stubs.fake_user_uuid,
 109             }
 110             stubs.api.collections().list().execute.return_value = {"items": []}
 111             stubs.api.containers().current().execute.return_value = {
 112                 "uuid": stubs.fake_container_uuid,
 113             }
 114             stubs.api.config()["StorageClasses"].items.return_value = {
 115                 "default": {
 116                     "Default": True
 117                 }
 118             }.items()
 119
 120             class CollectionExecute(object):
 121                 def __init__(self, exe):
 122                     self.exe = exe
 123                 def execute(self, num_retries=None):
 124                     return self.exe
 125
 126             def collection_createstub(created_collections, body, ensure_unique_name=None):
 127                 mt = body["manifest_text"].encode('utf-8')
 128                 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
 129                 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
 130                 created_collections[uuid] = {
 131                     "uuid": uuid,
 132                     "portable_data_hash": pdh,
 133                     "manifest_text": mt.decode('utf-8')
 134                 }
 135                 return CollectionExecute(created_collections[uuid])
 136
 137             def collection_getstub(created_collections, uuid):
 138                 for v in viewvalues(created_collections):
 139                     if uuid in (v["uuid"], v["portable_data_hash"]):
 140                         return CollectionExecute(v)
 141
 142             created_collections = {
 143                 "99999999999999999999999999999998+99": {
 144                     "uuid": "",
 145                     "portable_data_hash": "99999999999999999999999999999998+99",
 146                     "manifest_text": ". 99999999999999999999999999999998+99 0:0:file1.txt"
 147                 },
 148                 "99999999999999999999999999999997+99": {
 149                     "uuid": "",
 150                     "portable_data_hash": "99999999999999999999999999999997+99",
 151                     "manifest_text": ". 99999999999999999999999999999997+99 0:0:file1.txt"
 152                 },
 153                 "99999999999999999999999999999994+99": {
 154                     "uuid": "",
 155                     "portable_data_hash": "99999999999999999999999999999994+99",
 156                     "manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
 157                 },
 158                 "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
 159                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
 160                     "portable_data_hash": "999999999999999999999999999999d3+99",
 161                     "manifest_text": ""
 162                 },
 163                 "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
 164                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
 165                     "portable_data_hash": "999999999999999999999999999999d4+99",
 166                     "manifest_text": ""
 167                 },
 168                 "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
 169                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
 170                     "portable_data_hash": "999999999999999999999999999999d5+99",
 171                     "manifest_text": ""
 172                 },
 173                 "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
 174                     "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
 175                     "portable_data_hash": "999999999999999999999999999999d6+99",
 176                     "manifest_text": ""
 177                 }
 178             }
 179             stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
 180             stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
 181
 182             stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 183             stubs.api.jobs().create().execute.return_value = {
 184                 "uuid": stubs.expect_job_uuid,
 185                 "state": "Queued",
 186             }
 187
 188             stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 189             stubs.api.container_requests().create().execute.return_value = {
 190                 "uuid": stubs.expect_container_request_uuid,
 191                 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
 192                 "state": "Queued"
 193             }
 194
 195             stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 196             stubs.api.pipeline_templates().create().execute.return_value = {
 197                 "uuid": stubs.expect_pipeline_template_uuid,
 198             }
 199             stubs.expect_job_spec = {
 200                 'runtime_constraints': {
 201                     'docker_image': '999999999999999999999999999999d3+99',
 202                     'min_ram_mb_per_node': 1024
 203                 },
 204                 'script_parameters': {
 205                     'x': {
 206                         'basename': 'blorp.txt',
 207                         'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 208                         'class': 'File'
 209                     },
 210                     'y': {
 211                         'basename': '99999999999999999999999999999998+99',
 212                         'location': 'keep:99999999999999999999999999999998+99',
 213                         'class': 'Directory'
 214                     },
 215                     'z': {
 216                         'basename': 'anonymous',
 217                         "listing": [{
 218                             "basename": "renamed.txt",
 219                             "class": "File",
 220                             "location": "keep:99999999999999999999999999999998+99/file1.txt",
 221                             "size": 0
 222                         }],
 223                         'class': 'Directory'
 224                     },
 225                     'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
 226                 },
 227                 'repository': 'arvados',
 228                 'script_version': 'master',
 229                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 230                 'script': 'cwl-runner'
 231             }
 232             stubs.pipeline_component = stubs.expect_job_spec.copy()
 233             stubs.expect_pipeline_instance = {
 234                 'name': 'submit_wf.cwl',
 235                 'state': 'RunningOnServer',
 236                 'owner_uuid': None,
 237                 "components": {
 238                     "cwl-runner": {
 239                         'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
 240                         'script_parameters': {
 241                             'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 242                             'x': {"value": {
 243                                 'basename': 'blorp.txt',
 244                                 'class': 'File',
 245                                 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 246                                 "size": 16
 247                             }},
 248                             'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 249                                   'listing': [
 250                                       {
 251                                           'basename': 'renamed.txt',
 252                                           'class': 'File', 'location':
 253                                           'keep:99999999999999999999999999999998+99/file1.txt',
 254                                           'size': 0
 255                                       }
 256                                   ]}},
 257                             'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
 258                             'arv:debug': True,
 259                             'arv:enable_reuse': True,
 260                             'arv:on_error': 'continue'
 261                         },
 262                         'repository': 'arvados',
 263                         'script_version': 'master',
 264                         'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 265                         'script': 'cwl-runner',
 266                         'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 267                     }
 268                 }
 269             }
 270             stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 271             stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 272             stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 273             stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 274             stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 275                 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 276                 "state": "Queued"
 277             }
 278             stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 279             stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 280
 281             cwd = os.getcwd()
 282             filepath = os.path.join(cwd, "tests/wf/submit_wf_packed.cwl")
 283             with open(filepath) as f:
 284                 expect_packed_workflow = yaml.round_trip_load(f)
 285
 286             if wfpath is None:
 287                 wfpath = wfname
 288
 289             gitinfo_workflow = copy.deepcopy(expect_packed_workflow)
 290             gitinfo_workflow["$graph"][0]["id"] = "file://%s/tests/wf/%s" % (cwd, wfpath)
 291             mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow)
 292
 293             stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool)
 294             expect_packed_workflow.update(stubs.git_info)
 295
 296             stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
 297
 298             if wfname == wfpath:
 299                 container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
 300             else:
 301                 container_name = wfname
 302
 303             stubs.expect_container_spec = {
 304                 'priority': 500,
 305                 'mounts': {
 306                     '/var/spool/cwl': {
 307                         'writable': True,
 308                         'kind': 'collection'
 309                     },
 310                     '/var/lib/cwl/workflow.json': {
 311                         'content': expect_packed_workflow,
 312                         'kind': 'json'
 313                     },
 314                     'stdout': {
 315                         'path': '/var/spool/cwl/cwl.output.json',
 316                         'kind': 'file'
 317                     },
 318                     '/var/lib/cwl/cwl.input.json': {
 319                         'kind': 'json',
 320                         'content': {
 321                             'y': {
 322                                 'basename': '99999999999999999999999999999998+99',
 323                                 'location': 'keep:99999999999999999999999999999998+99',
 324                                 'class': 'Directory'},
 325                             'x': {
 326                                 'basename': u'blorp.txt',
 327                                 'class': 'File',
 328                                 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
 329                                 "size": 16
 330                             },
 331                             'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 332                                 {'basename': 'renamed.txt',
 333                                  'class': 'File',
 334                                  'location': 'keep:99999999999999999999999999999998+99/file1.txt',
 335                                  'size': 0
 336                                 }
 337                             ]}
 338                         },
 339                         'kind': 'json'
 340                     }
 341                 },
 342                 'secret_mounts': {},
 343                 'state': 'Committed',
 344                 'command': ['arvados-cwl-runner', '--local', '--api=containers',
 345                             '--no-log-timestamps', '--disable-validate', '--disable-color',
 346                             '--eval-timeout=20', '--thread-count=0',
 347                             '--enable-reuse', "--collection-cache-size=256",
 348                             '--output-name=Output from workflow '+container_name,
 349                             '--debug', '--on-error=continue',
 350                             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 351                 'name': container_name,
 352                 'container_image': '999999999999999999999999999999d3+99',
 353                 'output_name': 'Output from workflow %s' % (container_name),
 354                 'output_path': '/var/spool/cwl',
 355                 'cwd': '/var/spool/cwl',
 356                 'runtime_constraints': {
 357                     'API': True,
 358                     'vcpus': 1,
 359                     'ram': (1024+256)*1024*1024
 360                 },
 361                 'use_existing': False,
 362                 'properties': stubs.git_props,
 363                 'secret_mounts': {}
 364             }
 365
 366             stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 367             stubs.api.workflows().create().execute.return_value = {
 368                 "uuid": stubs.expect_workflow_uuid,
 369             }
 370             def update_mock(**kwargs):
 371                 stubs.updated_uuid = kwargs.get('uuid')
 372                 return mock.DEFAULT
 373             stubs.api.workflows().update.side_effect = update_mock
 374             stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 375                 "uuid": stubs.updated_uuid,
 376             }
 377
 378             return func(self, stubs, *args, **kwargs)
 379         return wrapped
 380     return outer_wrapper
 381
 382 class TestSubmit(unittest.TestCase):
 383
 384     def setUp(self):
 385         cwltool.process._names = set()
 386         arvados_cwl.arvdocker.arv_docker_clear_cache()
 387
 388     def tearDown(self):
 389         root_logger = logging.getLogger('')
 390
 391         # Remove existing RuntimeStatusLoggingHandlers if they exist
 392         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
 393         root_logger.handlers = handlers
 394
 395     @mock.patch("time.sleep")
 396     @stubs()
 397     def test_submit_invalid_runner_ram(self, stubs, tm):
 398         exited = arvados_cwl.main(
 399             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 400              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 401             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 402         self.assertEqual(exited, 1)
 403
 404
 405     @stubs()
 406     def test_submit_container(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--submit", "--no-wait", "--api=containers", "--debug",
 409                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 410             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 411
 412         stubs.api.collections().create.assert_has_calls([
 413             mock.call(body=JsonDiffMatcher({
 414                 'manifest_text':
 415                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 416                 'replication_desired': None,
 417                 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)',
 418             }), ensure_unique_name=False),
 419             mock.call(body=JsonDiffMatcher({
 420                 'manifest_text':
 421                 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
 422                 'replication_desired': None,
 423                 'name': 'submit_tool.cwl dependencies (5d373e7629203ce39e7c22af98a0f881+52)',
 424             }), ensure_unique_name=False),
 425             ])
 426
 427         expect_container = copy.deepcopy(stubs.expect_container_spec)
 428         stubs.api.container_requests().create.assert_called_with(
 429             body=JsonDiffMatcher(expect_container))
 430         self.assertEqual(stubs.capture_stdout.getvalue(),
 431                          stubs.expect_container_request_uuid + '\n')
 432         self.assertEqual(exited, 0)
 433
 434
 435     @stubs()
 436     def test_submit_container_tool(self, stubs):
 437         # test for issue #16139
 438         exited = arvados_cwl.main(
 439             ["--submit", "--no-wait", "--api=containers", "--debug",
 440                 "tests/tool/tool_with_sf.cwl", "tests/tool/tool_with_sf.yml"],
 441             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 442
 443         self.assertEqual(stubs.capture_stdout.getvalue(),
 444                          stubs.expect_container_request_uuid + '\n')
 445         self.assertEqual(exited, 0)
 446
 447     @stubs()
 448     def test_submit_container_no_reuse(self, stubs):
 449         exited = arvados_cwl.main(
 450             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 451                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 452             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 453
 454         expect_container = copy.deepcopy(stubs.expect_container_spec)
 455         expect_container["command"] = [
 456             'arvados-cwl-runner', '--local', '--api=containers',
 457             '--no-log-timestamps', '--disable-validate', '--disable-color',
 458             '--eval-timeout=20', '--thread-count=0',
 459             '--disable-reuse', "--collection-cache-size=256",
 460             '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 461             '--debug', '--on-error=continue',
 462             '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 463         expect_container["use_existing"] = False
 464
 465         stubs.api.container_requests().create.assert_called_with(
 466             body=JsonDiffMatcher(expect_container))
 467         self.assertEqual(stubs.capture_stdout.getvalue(),
 468                          stubs.expect_container_request_uuid + '\n')
 469         self.assertEqual(exited, 0)
 470
 471     @stubs(('submit_wf_no_reuse.cwl', None))
 472     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
 473         exited = arvados_cwl.main(
 474             ["--submit", "--no-wait", "--api=containers", "--debug",
 475              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
 476             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 477         self.assertEqual(exited, 0)
 478
 479         expect_container = copy.deepcopy(stubs.expect_container_spec)
 480         expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
 481         expect_container["use_existing"] = False
 482         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
 483             {
 484                 "class": "http://arvados.org/cwl#ReuseRequirement",
 485                 "enableReuse": False,
 486             },
 487         ]
 488         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
 489             "arv": "http://arvados.org/cwl#",
 490             "cwltool": "http://commonwl.org/cwltool#"
 491         }
 492
 493         stubs.api.container_requests().create.assert_called_with(
 494             body=JsonDiffMatcher(expect_container))
 495         self.assertEqual(stubs.capture_stdout.getvalue(),
 496                          stubs.expect_container_request_uuid + '\n')
 497
 498
 499     @stubs()
 500     def test_submit_container_on_error(self, stubs):
 501         exited = arvados_cwl.main(
 502             ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
 503                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 504             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 505
 506         expect_container = copy.deepcopy(stubs.expect_container_spec)
 507         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 508                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 509                                        '--eval-timeout=20', '--thread-count=0',
 510                                        '--enable-reuse', "--collection-cache-size=256",
 511                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 512                                        '--debug', '--on-error=stop',
 513                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 514
 515         stubs.api.container_requests().create.assert_called_with(
 516             body=JsonDiffMatcher(expect_container))
 517         self.assertEqual(stubs.capture_stdout.getvalue(),
 518                          stubs.expect_container_request_uuid + '\n')
 519         self.assertEqual(exited, 0)
 520
 521     @stubs()
 522     def test_submit_container_output_name(self, stubs):
 523         output_name = "test_output_name"
 524
 525         exited = arvados_cwl.main(
 526             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 527                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 528             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 529
 530         expect_container = copy.deepcopy(stubs.expect_container_spec)
 531         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 532                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 533                                        '--eval-timeout=20', '--thread-count=0',
 534                                        '--enable-reuse', "--collection-cache-size=256",
 535                                        "--output-name="+output_name, '--debug', '--on-error=continue',
 536                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 537         expect_container["output_name"] = output_name
 538
 539         stubs.api.container_requests().create.assert_called_with(
 540             body=JsonDiffMatcher(expect_container))
 541         self.assertEqual(stubs.capture_stdout.getvalue(),
 542                          stubs.expect_container_request_uuid + '\n')
 543         self.assertEqual(exited, 0)
 544
 545     @stubs()
 546     def test_submit_storage_classes(self, stubs):
 547         exited = arvados_cwl.main(
 548             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
 549                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 550             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 551
 552         expect_container = copy.deepcopy(stubs.expect_container_spec)
 553         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 554                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 555                                        '--eval-timeout=20', '--thread-count=0',
 556                                        '--enable-reuse', "--collection-cache-size=256",
 557                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 558                                        "--debug",
 559                                        "--storage-classes=foo", '--on-error=continue',
 560                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 561
 562         stubs.api.container_requests().create.assert_called_with(
 563             body=JsonDiffMatcher(expect_container))
 564         self.assertEqual(stubs.capture_stdout.getvalue(),
 565                          stubs.expect_container_request_uuid + '\n')
 566         self.assertEqual(exited, 0)
 567
 568     @stubs()
 569     def test_submit_multiple_storage_classes(self, stubs):
 570         exited = arvados_cwl.main(
 571             ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo,bar", "--intermediate-storage-classes=baz",
 572                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 573             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 574
 575         expect_container = copy.deepcopy(stubs.expect_container_spec)
 576         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 577                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 578                                        '--eval-timeout=20', '--thread-count=0',
 579                                        '--enable-reuse', "--collection-cache-size=256",
 580                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 581                                        "--debug",
 582                                        "--storage-classes=foo,bar", "--intermediate-storage-classes=baz", '--on-error=continue',
 583                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 584
 585         stubs.api.container_requests().create.assert_called_with(
 586             body=JsonDiffMatcher(expect_container))
 587         self.assertEqual(stubs.capture_stdout.getvalue(),
 588                          stubs.expect_container_request_uuid + '\n')
 589         self.assertEqual(exited, 0)
 590
 591     @mock.patch("cwltool.task_queue.TaskQueue")
 592     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 593     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 594     @stubs()
 595     def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 596         final_output_c = arvados.collection.Collection()
 597         make_output.return_value = ({},final_output_c)
 598
 599         def set_final_output(job_order, output_callback, runtimeContext):
 600             output_callback({"out": "zzzzz"}, "success")
 601             return []
 602         job.side_effect = set_final_output
 603
 604         exited = arvados_cwl.main(
 605             ["--debug", "--local", "--storage-classes=foo", "--disable-git",
 606                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 607             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 608
 609         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['foo'], '', {}, {"out": "zzzzz"})
 610         self.assertEqual(exited, 0)
 611
 612     @mock.patch("cwltool.task_queue.TaskQueue")
 613     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 614     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 615     @stubs()
 616     def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
 617         final_output_c = arvados.collection.Collection()
 618         make_output.return_value = ({},final_output_c)
 619         stubs.api.config().get.return_value = {"default": {"Default": True}}
 620
 621         def set_final_output(job_order, output_callback, runtimeContext):
 622             output_callback({"out": "zzzzz"}, "success")
 623             return []
 624         job.side_effect = set_final_output
 625
 626         exited = arvados_cwl.main(
 627             ["--debug", "--local", "--disable-git",
 628                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 629             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 630
 631         make_output.assert_called_with(u'Output from workflow submit_wf.cwl', ['default'], '', {}, {"out": "zzzzz"})
 632         self.assertEqual(exited, 0)
 633
 634     @mock.patch("cwltool.task_queue.TaskQueue")
 635     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
 636     @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
 637     @stubs()
 638     def test_storage_class_hint_to_make_output_collection(self, stubs, make_output, job, tq):
 639         final_output_c = arvados.collection.Collection()
 640         make_output.return_value = ({},final_output_c)
 641
 642         def set_final_output(job_order, output_callback, runtimeContext):
 643             output_callback({"out": "zzzzz"}, "success")
 644             return []
 645         job.side_effect = set_final_output
 646
 647         exited = arvados_cwl.main(
 648             ["--debug", "--local", "--disable-git",
 649                 "tests/wf/submit_storage_class_wf.cwl", "tests/submit_test_job.json"],
 650             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 651
 652         make_output.assert_called_with(u'Output from workflow submit_storage_class_wf.cwl', ['foo', 'bar'], '', {}, {"out": "zzzzz"})
 653         self.assertEqual(exited, 0)
 654
 655     @stubs()
 656     def test_submit_container_output_ttl(self, stubs):
 657         exited = arvados_cwl.main(
 658             ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
 659                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 660             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 661
 662         expect_container = copy.deepcopy(stubs.expect_container_spec)
 663         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 664                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 665                                        '--eval-timeout=20', '--thread-count=0',
 666                                        '--enable-reuse', "--collection-cache-size=256",
 667                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 668                                        '--debug',
 669                                        '--on-error=continue',
 670                                        "--intermediate-output-ttl=3600",
 671                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 672
 673         stubs.api.container_requests().create.assert_called_with(
 674             body=JsonDiffMatcher(expect_container))
 675         self.assertEqual(stubs.capture_stdout.getvalue(),
 676                          stubs.expect_container_request_uuid + '\n')
 677         self.assertEqual(exited, 0)
 678
 679     @stubs()
 680     def test_submit_container_trash_intermediate(self, stubs):
 681         exited = arvados_cwl.main(
 682             ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
 683                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 684             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 685
 686
 687         expect_container = copy.deepcopy(stubs.expect_container_spec)
 688         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 689                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 690                                        '--eval-timeout=20', '--thread-count=0',
 691                                        '--enable-reuse', "--collection-cache-size=256",
 692                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 693                                        '--debug', '--on-error=continue',
 694                                        "--trash-intermediate",
 695                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 696
 697         stubs.api.container_requests().create.assert_called_with(
 698             body=JsonDiffMatcher(expect_container))
 699         self.assertEqual(stubs.capture_stdout.getvalue(),
 700                          stubs.expect_container_request_uuid + '\n')
 701         self.assertEqual(exited, 0)
 702
 703     @stubs()
 704     def test_submit_container_output_tags(self, stubs):
 705         output_tags = "tag0,tag1,tag2"
 706
 707         exited = arvados_cwl.main(
 708             ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 709                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 710             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 711
 712         expect_container = copy.deepcopy(stubs.expect_container_spec)
 713         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 714                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 715                                        '--eval-timeout=20', '--thread-count=0',
 716                                        '--enable-reuse', "--collection-cache-size=256",
 717                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 718                                        "--output-tags="+output_tags, '--debug', '--on-error=continue',
 719                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 720
 721         stubs.api.container_requests().create.assert_called_with(
 722             body=JsonDiffMatcher(expect_container))
 723         self.assertEqual(stubs.capture_stdout.getvalue(),
 724                          stubs.expect_container_request_uuid + '\n')
 725         self.assertEqual(exited, 0)
 726
 727     @stubs()
 728     def test_submit_container_runner_ram(self, stubs):
 729         exited = arvados_cwl.main(
 730             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 731                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 732             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 733
 734         expect_container = copy.deepcopy(stubs.expect_container_spec)
 735         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 736
 737         stubs.api.container_requests().create.assert_called_with(
 738             body=JsonDiffMatcher(expect_container))
 739         self.assertEqual(stubs.capture_stdout.getvalue(),
 740                          stubs.expect_container_request_uuid + '\n')
 741         self.assertEqual(exited, 0)
 742
 743     @mock.patch("arvados.collection.CollectionReader")
 744     @mock.patch("time.sleep")
 745     @stubs()
 746     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 747         collectionReader().exists.return_value = True
 748         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
 749         exited = arvados_cwl.main(
 750             ["--submit", "--no-wait", "--api=containers", "--debug",
 751              "tests/wf/submit_keepref_wf.cwl"],
 752             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 753         self.assertEqual(exited, 0)
 754
 755     @mock.patch("arvados.collection.CollectionReader")
 756     @mock.patch("time.sleep")
 757     @stubs()
 758     def test_submit_keepref(self, stubs, tm, reader):
 759         with open("tests/wf/expect_arvworkflow.cwl") as f:
 760             reader().open().__enter__().read.return_value = f.read()
 761
 762         exited = arvados_cwl.main(
 763             ["--submit", "--no-wait", "--api=containers", "--debug",
 764              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 765             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 766
 767         expect_container = {
 768             'priority': 500,
 769             'mounts': {
 770                 '/var/spool/cwl': {
 771                     'writable': True,
 772                     'kind': 'collection'
 773                 },
 774                 'stdout': {
 775                     'path': '/var/spool/cwl/cwl.output.json',
 776                     'kind': 'file'
 777                 },
 778                 '/var/lib/cwl/workflow': {
 779                     'portable_data_hash': '99999999999999999999999999999994+99',
 780                     'kind': 'collection'
 781                 },
 782                 '/var/lib/cwl/cwl.input.json': {
 783                     'content': {
 784                         'x': 'XxX'
 785                     },
 786                     'kind': 'json'
 787                 }
 788             }, 'state': 'Committed',
 789             'output_path': '/var/spool/cwl',
 790             'name': 'expect_arvworkflow.cwl#main',
 791             'output_name': 'Output from workflow expect_arvworkflow.cwl#main',
 792             'container_image': '999999999999999999999999999999d3+99',
 793             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 794                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 795                         '--eval-timeout=20', '--thread-count=0',
 796                         '--enable-reuse', "--collection-cache-size=256",
 797                         '--output-name=Output from workflow expect_arvworkflow.cwl#main',
 798                         '--debug', '--on-error=continue',
 799                         '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 800             'cwd': '/var/spool/cwl',
 801             'runtime_constraints': {
 802                 'API': True,
 803                 'vcpus': 1,
 804                 'ram': 1342177280
 805             },
 806             'use_existing': False,
 807             'properties': {},
 808             'secret_mounts': {}
 809         }
 810
 811         stubs.api.container_requests().create.assert_called_with(
 812             body=JsonDiffMatcher(expect_container))
 813         self.assertEqual(stubs.capture_stdout.getvalue(),
 814                          stubs.expect_container_request_uuid + '\n')
 815         self.assertEqual(exited, 0)
 816
 817     @mock.patch("time.sleep")
 818     @stubs()
 819     def test_submit_arvworkflow(self, stubs, tm):
 820         with open("tests/wf/expect_arvworkflow.cwl") as f:
 821             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 822
 823         exited = arvados_cwl.main(
 824             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
 825              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 826             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 827
 828         expect_container = {
 829             'priority': 500,
 830             'mounts': {
 831                 '/var/spool/cwl': {
 832                     'writable': True,
 833                     'kind': 'collection'
 834                 },
 835                 'stdout': {
 836                     'path': '/var/spool/cwl/cwl.output.json',
 837                     'kind': 'file'
 838                 },
 839                 '/var/lib/cwl/workflow.json': {
 840                     'kind': 'json',
 841                     'content': {
 842                         'cwlVersion': 'v1.0',
 843                         '$graph': [
 844                             {
 845                                 'id': '#main',
 846                                 'inputs': [
 847                                     {'type': 'string', 'id': '#main/x'}
 848                                 ],
 849                                 'steps': [
 850                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 851                                      'run': '#submit_tool.cwl',
 852                                      'id': '#main/step1',
 853                                      'out': []}
 854                                 ],
 855                                 'class': 'Workflow',
 856                                 'outputs': []
 857                             },
 858                             {
 859                                 'inputs': [
 860                                     {
 861                                         'inputBinding': {'position': 1},
 862                                         'type': 'string',
 863                                         'id': '#submit_tool.cwl/x'}
 864                                 ],
 865                                 'requirements': [
 866                                     {
 867                                         'dockerPull': 'debian:buster-slim',
 868                                         'class': 'DockerRequirement'
 869                                     }
 870                                 ],
 871                                 'id': '#submit_tool.cwl',
 872                                 'outputs': [],
 873                                 'baseCommand': 'cat',
 874                                 'class': 'CommandLineTool'
 875                             }
 876                         ]
 877                     }
 878                 },
 879                 '/var/lib/cwl/cwl.input.json': {
 880                     'content': {
 881                         'x': 'XxX'
 882                     },
 883                     'kind': 'json'
 884                 }
 885             }, 'state': 'Committed',
 886             'output_path': '/var/spool/cwl',
 887             'name': 'a test workflow',
 888             'container_image': "999999999999999999999999999999d3+99",
 889             'command': ['arvados-cwl-runner', '--local', '--api=containers',
 890                         '--no-log-timestamps', '--disable-validate', '--disable-color',
 891                         '--eval-timeout=20', '--thread-count=0',
 892                         '--enable-reuse', "--collection-cache-size=256",
 893                         "--output-name=Output from workflow a test workflow",
 894                         '--debug', '--on-error=continue',
 895                         '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 896             'output_name': 'Output from workflow a test workflow',
 897             'cwd': '/var/spool/cwl',
 898             'runtime_constraints': {
 899                 'API': True,
 900                 'vcpus': 1,
 901                 'ram': 1342177280
 902             },
 903             'use_existing': False,
 904             'properties': {
 905                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 906             },
 907             'secret_mounts': {}
 908         }
 909
 910         stubs.api.container_requests().create.assert_called_with(
 911             body=JsonDiffMatcher(expect_container))
 912         self.assertEqual(stubs.capture_stdout.getvalue(),
 913                          stubs.expect_container_request_uuid + '\n')
 914         self.assertEqual(exited, 0)
 915
 916     @stubs(('hello container 123', 'submit_wf.cwl'))
 917     def test_submit_container_name(self, stubs):
 918         exited = arvados_cwl.main(
 919             ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 920                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 921             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 922
 923         expect_container = copy.deepcopy(stubs.expect_container_spec)
 924
 925         stubs.api.container_requests().create.assert_called_with(
 926             body=JsonDiffMatcher(expect_container))
 927         self.assertEqual(stubs.capture_stdout.getvalue(),
 928                          stubs.expect_container_request_uuid + '\n')
 929         self.assertEqual(exited, 0)
 930
 931     @stubs()
 932     def test_submit_missing_input(self, stubs):
 933         exited = arvados_cwl.main(
 934             ["--submit", "--no-wait", "--api=containers", "--debug",
 935              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 936             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 937         self.assertEqual(exited, 0)
 938
 939         exited = arvados_cwl.main(
 940             ["--submit", "--no-wait", "--api=containers", "--debug",
 941              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
 942             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 943         self.assertEqual(exited, 1)
 944
 945     @stubs()
 946     def test_submit_container_project(self, stubs):
 947         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 948         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
 949         exited = arvados_cwl.main(
 950             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
 951                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 952             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 953
 954         expect_container = copy.deepcopy(stubs.expect_container_spec)
 955         expect_container["owner_uuid"] = project_uuid
 956         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 957                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 958                                        "--eval-timeout=20", "--thread-count=0",
 959                                        '--enable-reuse', "--collection-cache-size=256",
 960                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 961                                        '--debug',
 962                                        '--on-error=continue',
 963                                        '--project-uuid='+project_uuid,
 964                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 965
 966         stubs.api.container_requests().create.assert_called_with(
 967             body=JsonDiffMatcher(expect_container))
 968         self.assertEqual(stubs.capture_stdout.getvalue(),
 969                          stubs.expect_container_request_uuid + '\n')
 970         self.assertEqual(exited, 0)
 971
 972     @stubs()
 973     def test_submit_container_eval_timeout(self, stubs):
 974         exited = arvados_cwl.main(
 975             ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
 976                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 977             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 978
 979         expect_container = copy.deepcopy(stubs.expect_container_spec)
 980         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
 981                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
 982                                        '--eval-timeout=60.0', '--thread-count=0',
 983                                        '--enable-reuse', "--collection-cache-size=256",
 984                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
 985                                        '--debug', '--on-error=continue',
 986                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
 987
 988         stubs.api.container_requests().create.assert_called_with(
 989             body=JsonDiffMatcher(expect_container))
 990         self.assertEqual(stubs.capture_stdout.getvalue(),
 991                          stubs.expect_container_request_uuid + '\n')
 992         self.assertEqual(exited, 0)
 993
 994     @stubs()
 995     def test_submit_container_collection_cache(self, stubs):
 996         exited = arvados_cwl.main(
 997             ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
 998                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 999             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1000
1001         expect_container = copy.deepcopy(stubs.expect_container_spec)
1002         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1003                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1004                                        '--eval-timeout=20', '--thread-count=0',
1005                                        '--enable-reuse', "--collection-cache-size=500",
1006                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1007                                        '--debug', '--on-error=continue',
1008                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1009         expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
1010
1011         stubs.api.container_requests().create.assert_called_with(
1012             body=JsonDiffMatcher(expect_container))
1013         self.assertEqual(stubs.capture_stdout.getvalue(),
1014                          stubs.expect_container_request_uuid + '\n')
1015         self.assertEqual(exited, 0)
1016
1017     @stubs()
1018     def test_submit_container_thread_count(self, stubs):
1019         exited = arvados_cwl.main(
1020             ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
1021                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1022             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1023
1024         expect_container = copy.deepcopy(stubs.expect_container_spec)
1025         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
1026                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1027                                        '--eval-timeout=20', '--thread-count=20',
1028                                        '--enable-reuse', "--collection-cache-size=256",
1029                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1030                                        '--debug', '--on-error=continue',
1031                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1032
1033         stubs.api.container_requests().create.assert_called_with(
1034             body=JsonDiffMatcher(expect_container))
1035         self.assertEqual(stubs.capture_stdout.getvalue(),
1036                          stubs.expect_container_request_uuid + '\n')
1037         self.assertEqual(exited, 0)
1038
1039     @stubs()
1040     def test_submit_container_runner_image(self, stubs):
1041         exited = arvados_cwl.main(
1042             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
1043                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1044             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1045
1046         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
1047
1048         expect_container = copy.deepcopy(stubs.expect_container_spec)
1049         stubs.api.container_requests().create.assert_called_with(
1050             body=JsonDiffMatcher(expect_container))
1051         self.assertEqual(stubs.capture_stdout.getvalue(),
1052                          stubs.expect_container_request_uuid + '\n')
1053         self.assertEqual(exited, 0)
1054
1055     @stubs()
1056     def test_submit_priority(self, stubs):
1057         exited = arvados_cwl.main(
1058             ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
1059                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1060             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1061
1062         stubs.expect_container_spec["priority"] = 669
1063
1064         expect_container = copy.deepcopy(stubs.expect_container_spec)
1065         stubs.api.container_requests().create.assert_called_with(
1066             body=JsonDiffMatcher(expect_container))
1067         self.assertEqual(stubs.capture_stdout.getvalue(),
1068                          stubs.expect_container_request_uuid + '\n')
1069         self.assertEqual(exited, 0)
1070
1071     @stubs(('submit_wf_runner_resources.cwl', None))
1072     def test_submit_wf_runner_resources(self, stubs):
1073         exited = arvados_cwl.main(
1074             ["--submit", "--no-wait", "--api=containers", "--debug",
1075                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
1076             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1077
1078         expect_container = copy.deepcopy(stubs.expect_container_spec)
1079         expect_container["runtime_constraints"] = {
1080             "API": True,
1081             "vcpus": 2,
1082             "ram": (2000+512) * 2**20
1083         }
1084         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1085             {
1086                 "class": "http://arvados.org/cwl#WorkflowRunnerResources",
1087                 "coresMin": 2,
1088                 "ramMin": 2000,
1089                 "keep_cache": 512
1090             }
1091         ]
1092         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1093             "arv": "http://arvados.org/cwl#",
1094         }
1095         expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
1096
1097         stubs.api.container_requests().create.assert_called_with(
1098             body=JsonDiffMatcher(expect_container))
1099         self.assertEqual(stubs.capture_stdout.getvalue(),
1100                          stubs.expect_container_request_uuid + '\n')
1101         self.assertEqual(exited, 0)
1102
1103     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
1104     @mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
1105     @mock.patch("arvados.api")
1106     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
1107         arvados_cwl.arvdocker.arv_docker_clear_cache()
1108
1109         arvrunner = mock.MagicMock()
1110         arvrunner.project_uuid = ""
1111         api.return_value = mock.MagicMock()
1112         arvrunner.api = api.return_value
1113         arvrunner.runtimeContext.match_local_docker = False
1114         arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
1115                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1116                                                                         "link_class": "docker_image_repo+tag",
1117                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1118                                                                         "owner_uuid": "",
1119                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1120                                                             {"items": [{"created_at": "",
1121                                                                         "head_uuid": "",
1122                                                                         "link_class": "docker_image_hash",
1123                                                                         "name": "123456",
1124                                                                         "owner_uuid": "",
1125                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1126                                                             {"items": [{"created_at": "",
1127                                                                         "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1128                                                                         "link_class": "docker_image_repo+tag",
1129                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
1130                                                                         "owner_uuid": "",
1131                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
1132                                                             {"items": [{"created_at": "",
1133                                                                         "head_uuid": "",
1134                                                                         "link_class": "docker_image_hash",
1135                                                                         "name": "123456",
1136                                                                         "owner_uuid": "",
1137                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
1138         )
1139         find_one_image_hash.return_value = "123456"
1140
1141         arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1142                                                                               "owner_uuid": "",
1143                                                                               "manifest_text": "",
1144                                                                               "properties": ""
1145                                                                               }], "items_available": 1, "offset": 0},
1146                                                                   {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1147                                                                               "owner_uuid": "",
1148                                                                               "manifest_text": "",
1149                                                                               "properties": ""
1150                                                                           }], "items_available": 1, "offset": 0})
1151         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
1152         arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
1153                                                                   "portable_data_hash": "9999999999999999999999999999999b+99"}
1154
1155         self.assertEqual("9999999999999999999999999999999b+99",
1156                          arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__, arvrunner.runtimeContext))
1157
1158
1159     @stubs()
1160     def test_submit_secrets(self, stubs):
1161         exited = arvados_cwl.main(
1162             ["--submit", "--no-wait", "--api=containers", "--debug",
1163                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
1164             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1165
1166         stubs.git_props["arv:gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1167         stubs.git_info["http://arvados.org/cwl#gitPath"] = "sdk/cwl/tests/wf/secret_wf.cwl"
1168
1169         expect_container = {
1170             "command": [
1171                 "arvados-cwl-runner",
1172                 "--local",
1173                 "--api=containers",
1174                 "--no-log-timestamps",
1175                 "--disable-validate",
1176                 "--disable-color",
1177                 "--eval-timeout=20",
1178                 '--thread-count=0',
1179                 "--enable-reuse",
1180                 "--collection-cache-size=256",
1181                 '--output-name=Output from workflow secret_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1182                 "--debug",
1183                 "--on-error=continue",
1184                 "/var/lib/cwl/workflow.json#main",
1185                 "/var/lib/cwl/cwl.input.json"
1186             ],
1187             "container_image": "999999999999999999999999999999d3+99",
1188             "cwd": "/var/spool/cwl",
1189             "mounts": {
1190                 "/var/lib/cwl/cwl.input.json": {
1191                     "content": {
1192                         "pw": {
1193                             "$include": "/secrets/s0"
1194                         }
1195                     },
1196                     "kind": "json"
1197                 },
1198                 "/var/lib/cwl/workflow.json": {
1199                     "content": {
1200                         "$graph": [
1201                             {
1202                                 "arguments": [
1203                                     "md5sum",
1204                                     "example.conf"
1205                                 ],
1206                                 "class": "CommandLineTool",
1207                                 "hints": [
1208                                     {
1209                                         "class": "http://commonwl.org/cwltool#Secrets",
1210                                         "secrets": [
1211                                             "#secret_job.cwl/pw"
1212                                         ]
1213                                     }
1214                                 ],
1215                                 "id": "#secret_job.cwl",
1216                                 "inputs": [
1217                                     {
1218                                         "id": "#secret_job.cwl/pw",
1219                                         "type": "string"
1220                                     }
1221                                 ],
1222                                 "outputs": [
1223                                     {
1224                                         "id": "#secret_job.cwl/out",
1225                                         "type": "File",
1226                                         "outputBinding": {
1227                                               "glob": "hashed_example.txt"
1228                                         }
1229                                     }
1230                                 ],
1231                                 "stdout": "hashed_example.txt",
1232                                 "requirements": [
1233                                     {
1234                                         "class": "InitialWorkDirRequirement",
1235                                         "listing": [
1236                                             {
1237                                                 "entry": "username: user\npassword: $(inputs.pw)\n",
1238                                                 "entryname": "example.conf"
1239                                             }
1240                                         ]
1241                                     }
1242                                 ]
1243                             },
1244                             {
1245                                 "class": "Workflow",
1246                                 "hints": [
1247                                     {
1248                                         "class": "DockerRequirement",
1249                                         "dockerPull": "debian:buster-slim",
1250                                         "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
1251                                     },
1252                                     {
1253                                         "class": "http://commonwl.org/cwltool#Secrets",
1254                                         "secrets": [
1255                                             "#main/pw"
1256                                         ]
1257                                     }
1258                                 ],
1259                                 "id": "#main",
1260                                 "inputs": [
1261                                     {
1262                                         "id": "#main/pw",
1263                                         "type": "string"
1264                                     }
1265                                 ],
1266                                 "outputs": [
1267                                     {
1268                                         "id": "#main/out",
1269                                         "outputSource": "#main/step1/out",
1270                                         "type": "File"
1271                                     }
1272                                 ],
1273                                 "steps": [
1274                                     {
1275                                         "id": "#main/step1",
1276                                         "in": [
1277                                             {
1278                                                 "id": "#main/step1/pw",
1279                                                 "source": "#main/pw"
1280                                             }
1281                                         ],
1282                                         "out": [
1283                                             "#main/step1/out"
1284                                         ],
1285                                         "run": "#secret_job.cwl"
1286                                     }
1287                                 ]
1288                             }
1289                         ],
1290                         "$namespaces": {
1291                             "cwltool": "http://commonwl.org/cwltool#"
1292                         },
1293                         "cwlVersion": "v1.0"
1294                     },
1295                     "kind": "json"
1296                 },
1297                 "/var/spool/cwl": {
1298                     "kind": "collection",
1299                     "writable": True
1300                 },
1301                 "stdout": {
1302                     "kind": "file",
1303                     "path": "/var/spool/cwl/cwl.output.json"
1304                 }
1305             },
1306             "name": "secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1307             "output_name": "Output from workflow secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
1308             "output_path": "/var/spool/cwl",
1309             "priority": 500,
1310             "properties": stubs.git_props,
1311             "runtime_constraints": {
1312                 "API": True,
1313                 "ram": 1342177280,
1314                 "vcpus": 1
1315             },
1316             "secret_mounts": {
1317                 "/secrets/s0": {
1318                     "content": "blorp",
1319                     "kind": "text"
1320                 }
1321             },
1322             "state": "Committed",
1323             "use_existing": False
1324         }
1325
1326         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"].update(stubs.git_info)
1327
1328         stubs.api.container_requests().create.assert_called_with(
1329             body=JsonDiffMatcher(expect_container))
1330         self.assertEqual(stubs.capture_stdout.getvalue(),
1331                          stubs.expect_container_request_uuid + '\n')
1332         self.assertEqual(exited, 0)
1333
1334     @stubs()
1335     def test_submit_request_uuid(self, stubs):
1336         stubs.api._rootDesc["remoteHosts"]["zzzzz"] = "123"
1337         stubs.expect_container_request_uuid = "zzzzz-xvhdp-yyyyyyyyyyyyyyy"
1338
1339         stubs.api.container_requests().update().execute.return_value = {
1340             "uuid": stubs.expect_container_request_uuid,
1341             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
1342             "state": "Queued"
1343         }
1344
1345         exited = arvados_cwl.main(
1346             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
1347                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1348             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1349
1350         stubs.api.container_requests().update.assert_called_with(
1351             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
1352         self.assertEqual(stubs.capture_stdout.getvalue(),
1353                          stubs.expect_container_request_uuid + '\n')
1354         self.assertEqual(exited, 0)
1355
1356     @stubs()
1357     def test_submit_container_cluster_id(self, stubs):
1358         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1359
1360         exited = arvados_cwl.main(
1361             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
1362                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1363             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1364
1365         expect_container = copy.deepcopy(stubs.expect_container_spec)
1366
1367         stubs.api.container_requests().create.assert_called_with(
1368             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
1369         self.assertEqual(stubs.capture_stdout.getvalue(),
1370                          stubs.expect_container_request_uuid + '\n')
1371         self.assertEqual(exited, 0)
1372
1373     @stubs()
1374     def test_submit_validate_cluster_id(self, stubs):
1375         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
1376         exited = arvados_cwl.main(
1377             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
1378              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1379             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1380         self.assertEqual(exited, 1)
1381
1382     @stubs()
1383     def test_submit_validate_project_uuid(self, stubs):
1384         # Fails with bad cluster prefix
1385         exited = arvados_cwl.main(
1386             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzb-j7d0g-zzzzzzzzzzzzzzz",
1387              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1388             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1389         self.assertEqual(exited, 1)
1390
1391         # Project lookup fails
1392         stubs.api.groups().get().execute.side_effect = Exception("Bad project")
1393         exited = arvados_cwl.main(
1394             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-j7d0g-zzzzzzzzzzzzzzx",
1395              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1396             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1397         self.assertEqual(exited, 1)
1398
1399         # It should work this time because it is looking up a user (and only group is stubbed out to fail)
1400         exited = arvados_cwl.main(
1401             ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid=zzzzz-tpzed-zzzzzzzzzzzzzzx",
1402              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1403             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1404         self.assertEqual(exited, 0)
1405
1406
1407     @mock.patch("arvados.collection.CollectionReader")
1408     @stubs()
1409     def test_submit_uuid_inputs(self, stubs, collectionReader):
1410         collectionReader().exists.return_value = True
1411         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1412         def list_side_effect(**kwargs):
1413             m = mock.MagicMock()
1414             if "count" in kwargs:
1415                 m.execute.return_value = {"items": [
1416                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999998+99"}
1417                 ]}
1418             else:
1419                 m.execute.return_value = {"items": []}
1420             return m
1421         stubs.api.collections().list.side_effect = list_side_effect
1422
1423         exited = arvados_cwl.main(
1424             ["--submit", "--no-wait", "--api=containers", "--debug",
1425                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1426             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1427
1428         expect_container = copy.deepcopy(stubs.expect_container_spec)
1429         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['basename'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1430         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['y']['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1431         expect_container['mounts']['/var/lib/cwl/cwl.input.json']['content']['z']['listing'][0]['http://arvados.org/cwl#collectionUUID'] = 'zzzzz-4zz18-zzzzzzzzzzzzzzz'
1432
1433         stubs.api.collections().list.assert_has_calls([
1434             mock.call(count='none',
1435                       filters=[['uuid', 'in', ['zzzzz-4zz18-zzzzzzzzzzzzzzz']]],
1436                       select=['uuid', 'portable_data_hash'])])
1437         stubs.api.container_requests().create.assert_called_with(
1438             body=JsonDiffMatcher(expect_container))
1439         self.assertEqual(stubs.capture_stdout.getvalue(),
1440                          stubs.expect_container_request_uuid + '\n')
1441         self.assertEqual(exited, 0)
1442
1443     @stubs()
1444     def test_submit_mismatched_uuid_inputs(self, stubs):
1445         def list_side_effect(**kwargs):
1446             m = mock.MagicMock()
1447             if "count" in kwargs:
1448                 m.execute.return_value = {"items": [
1449                     {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999997+99"}
1450                 ]}
1451             else:
1452                 m.execute.return_value = {"items": []}
1453             return m
1454         stubs.api.collections().list.side_effect = list_side_effect
1455
1456         for infile in ("tests/submit_test_job_with_mismatched_uuids.json", "tests/submit_test_job_with_inconsistent_uuids.json"):
1457             capture_stderr = StringIO()
1458             cwltool_logger = logging.getLogger('cwltool')
1459             stderr_logger = logging.StreamHandler(capture_stderr)
1460             cwltool_logger.addHandler(stderr_logger)
1461
1462             try:
1463                 exited = arvados_cwl.main(
1464                     ["--submit", "--no-wait", "--api=containers", "--debug",
1465                         "tests/wf/submit_wf.cwl", infile],
1466                     stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1467
1468                 self.assertEqual(exited, 1)
1469                 self.assertRegex(
1470                     re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
1471                     r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
1472             finally:
1473                 cwltool_logger.removeHandler(stderr_logger)
1474
1475     @mock.patch("arvados.collection.CollectionReader")
1476     @stubs()
1477     def test_submit_unknown_uuid_inputs(self, stubs, collectionReader):
1478         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "file1.txt")
1479         capture_stderr = StringIO()
1480
1481         cwltool_logger = logging.getLogger('cwltool')
1482         stderr_logger = logging.StreamHandler(capture_stderr)
1483         cwltool_logger.addHandler(stderr_logger)
1484
1485         exited = arvados_cwl.main(
1486             ["--submit", "--no-wait", "--api=containers", "--debug",
1487                 "tests/wf/submit_wf.cwl", "tests/submit_test_job_with_uuids.json"],
1488             stubs.capture_stdout, capture_stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1489
1490         try:
1491             self.assertEqual(exited, 1)
1492             self.assertRegex(
1493                 capture_stderr.getvalue(),
1494                 r"Collection\s*uuid\s*zzzzz-4zz18-zzzzzzzzzzzzzzz\s*not\s*found")
1495         finally:
1496             cwltool_logger.removeHandler(stderr_logger)
1497
1498     @stubs(('submit_wf_process_properties.cwl', None))
1499     def test_submit_set_process_properties(self, stubs):
1500         exited = arvados_cwl.main(
1501             ["--submit", "--no-wait", "--api=containers", "--debug",
1502                 "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
1503             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1504
1505         expect_container = copy.deepcopy(stubs.expect_container_spec)
1506
1507         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
1508             {
1509                 "class": "http://arvados.org/cwl#ProcessProperties",
1510                 "processProperties": [
1511                     {"propertyName": "baz",
1512                      "propertyValue": "$(inputs.x.basename)"},
1513                     {"propertyName": "foo",
1514                      "propertyValue": "bar"},
1515                     {"propertyName": "quux",
1516                      "propertyValue": {
1517                          "q1": 1,
1518                          "q2": 2
1519                      }
1520                     }
1521                 ],
1522             }
1523         ]
1524         expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
1525             "arv": "http://arvados.org/cwl#"
1526         }
1527
1528         expect_container["properties"].update({
1529             "baz": "blorp.txt",
1530             "foo": "bar",
1531             "quux": {
1532                 "q1": 1,
1533                 "q2": 2
1534             }
1535         })
1536
1537         stubs.api.container_requests().create.assert_called_with(
1538             body=JsonDiffMatcher(expect_container))
1539         self.assertEqual(stubs.capture_stdout.getvalue(),
1540                          stubs.expect_container_request_uuid + '\n')
1541         self.assertEqual(exited, 0)
1542
1543
1544     @stubs()
1545     def test_submit_enable_preemptible(self, stubs):
1546         exited = arvados_cwl.main(
1547             ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
1548                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1549             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1550
1551         expect_container = copy.deepcopy(stubs.expect_container_spec)
1552         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1553                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1554                                        '--eval-timeout=20', '--thread-count=0',
1555                                        '--enable-reuse', "--collection-cache-size=256",
1556                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1557                                        '--debug', '--on-error=continue',
1558                                        '--enable-preemptible',
1559                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1560
1561         stubs.api.container_requests().create.assert_called_with(
1562             body=JsonDiffMatcher(expect_container))
1563         self.assertEqual(stubs.capture_stdout.getvalue(),
1564                          stubs.expect_container_request_uuid + '\n')
1565         self.assertEqual(exited, 0)
1566
1567     @stubs()
1568     def test_submit_disable_preemptible(self, stubs):
1569         exited = arvados_cwl.main(
1570             ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
1571                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1572             stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
1573
1574         expect_container = copy.deepcopy(stubs.expect_container_spec)
1575         expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
1576                                        '--no-log-timestamps', '--disable-validate', '--disable-color',
1577                                        '--eval-timeout=20', '--thread-count=0',
1578                                        '--enable-reuse', "--collection-cache-size=256",
1579                                        '--output-name=Output from workflow submit_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
1580                                        '--debug', '--on-error=continue',
1581                                        '--disable-preemptible',
1582                                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
1583
1584         stubs.api.container_requests().create.assert_called_with(
1585             body=JsonDiffMatcher(expect_container))
1586         self.assertEqual(stubs.capture_stdout.getvalue(),
1587                          stubs.expect_container_request_uuid + '\n')
1588         self.assertEqual(exited, 0)
1589
1590
1591 class TestCreateWorkflow(unittest.TestCase):
1592     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
1593     expect_workflow = StripYAMLComments(
1594         open("tests/wf/expect_upload_wrapper.cwl").read().rstrip())
1595     expect_workflow_altname = StripYAMLComments(
1596         open("tests/wf/expect_upload_wrapper_altname.cwl").read().rstrip())
1597
1598     def setUp(self):
1599         cwltool.process._names = set()
1600         arvados_cwl.arvdocker.arv_docker_clear_cache()
1601
1602     def tearDown(self):
1603         root_logger = logging.getLogger('')
1604
1605         # Remove existing RuntimeStatusLoggingHandlers if they exist
1606         handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)]
1607         root_logger.handlers = handlers
1608
1609     @stubs()
1610     def test_create(self, stubs):
1611         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1612         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1613
1614         exited = arvados_cwl.main(
1615             ["--create-workflow", "--debug",
1616              "--api=containers",
1617              "--project-uuid", project_uuid,
1618              "--disable-git",
1619              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1620             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1621
1622         stubs.api.pipeline_templates().create.refute_called()
1623         stubs.api.container_requests().create.refute_called()
1624
1625         body = {
1626             "workflow": {
1627                 "owner_uuid": project_uuid,
1628                 "name": "submit_wf.cwl",
1629                 "description": "",
1630                 "definition": self.expect_workflow,
1631             }
1632         }
1633         stubs.api.workflows().create.assert_called_with(
1634             body=JsonDiffMatcher(body))
1635
1636         self.assertEqual(stubs.capture_stdout.getvalue(),
1637                          stubs.expect_workflow_uuid + '\n')
1638         self.assertEqual(exited, 0)
1639
1640     @stubs()
1641     def test_create_name(self, stubs):
1642         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1643         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1644
1645         exited = arvados_cwl.main(
1646             ["--create-workflow", "--debug",
1647              "--api=containers",
1648              "--project-uuid", project_uuid,
1649              "--name", "testing 123",
1650              "--disable-git",
1651              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1652             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1653
1654         stubs.api.pipeline_templates().create.refute_called()
1655         stubs.api.container_requests().create.refute_called()
1656
1657         body = {
1658             "workflow": {
1659                 "owner_uuid": project_uuid,
1660                 "name": "testing 123",
1661                 "description": "",
1662                 "definition": self.expect_workflow_altname,
1663             }
1664         }
1665         stubs.api.workflows().create.assert_called_with(
1666             body=JsonDiffMatcher(body))
1667
1668         self.assertEqual(stubs.capture_stdout.getvalue(),
1669                          stubs.expect_workflow_uuid + '\n')
1670         self.assertEqual(exited, 0)
1671
1672
1673     @stubs()
1674     def test_update(self, stubs):
1675         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1676         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1677
1678         exited = arvados_cwl.main(
1679             ["--update-workflow", self.existing_workflow_uuid,
1680              "--debug",
1681              "--disable-git",
1682              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1683             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1684
1685         body = {
1686             "workflow": {
1687                 "name": "submit_wf.cwl",
1688                 "description": "",
1689                 "definition": self.expect_workflow,
1690                 "owner_uuid": project_uuid
1691             }
1692         }
1693         stubs.api.workflows().update.assert_called_with(
1694             uuid=self.existing_workflow_uuid,
1695             body=JsonDiffMatcher(body))
1696         self.assertEqual(stubs.capture_stdout.getvalue(),
1697                          self.existing_workflow_uuid + '\n')
1698         self.assertEqual(exited, 0)
1699
1700
1701     @stubs()
1702     def test_update_name(self, stubs):
1703         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1704         stubs.api.workflows().get().execute.return_value = {"owner_uuid": project_uuid}
1705
1706         exited = arvados_cwl.main(
1707             ["--update-workflow", self.existing_workflow_uuid,
1708              "--debug", "--name", "testing 123",
1709              "--disable-git",
1710              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1711             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1712
1713         body = {
1714             "workflow": {
1715                 "name": "testing 123",
1716                 "description": "",
1717                 "definition": self.expect_workflow_altname,
1718                 "owner_uuid": project_uuid
1719             }
1720         }
1721         stubs.api.workflows().update.assert_called_with(
1722             uuid=self.existing_workflow_uuid,
1723             body=JsonDiffMatcher(body))
1724         self.assertEqual(stubs.capture_stdout.getvalue(),
1725                          self.existing_workflow_uuid + '\n')
1726         self.assertEqual(exited, 0)
1727
1728     @stubs()
1729     def test_create_collection_per_tool(self, stubs):
1730         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1731         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1732
1733         exited = arvados_cwl.main(
1734             ["--create-workflow", "--debug",
1735              "--api=containers",
1736              "--project-uuid", project_uuid,
1737              "--disable-git",
1738              "tests/collection_per_tool/collection_per_tool.cwl"],
1739             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1740
1741         toolfile = "tests/collection_per_tool/collection_per_tool_wrapper.cwl"
1742         expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
1743
1744         body = {
1745             "workflow": {
1746                 "owner_uuid": project_uuid,
1747                 "name": "collection_per_tool.cwl",
1748                 "description": "",
1749                 "definition": expect_workflow,
1750             }
1751         }
1752         stubs.api.workflows().create.assert_called_with(
1753             body=JsonDiffMatcher(body))
1754
1755         self.assertEqual(stubs.capture_stdout.getvalue(),
1756                          stubs.expect_workflow_uuid + '\n')
1757         self.assertEqual(exited, 0)
1758
1759     @stubs()
1760     def test_create_with_imports(self, stubs):
1761         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1762         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1763
1764         exited = arvados_cwl.main(
1765             ["--create-workflow", "--debug",
1766              "--api=containers",
1767              "--project-uuid", project_uuid,
1768              "tests/wf/feddemo/feddemo.cwl"],
1769             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1770
1771         stubs.api.pipeline_templates().create.refute_called()
1772         stubs.api.container_requests().create.refute_called()
1773
1774         self.assertEqual(stubs.capture_stdout.getvalue(),
1775                          stubs.expect_workflow_uuid + '\n')
1776         self.assertEqual(exited, 0)
1777
1778     @stubs()
1779     def test_create_with_no_input(self, stubs):
1780         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1781         stubs.api.groups().get().execute.return_value = {"group_class": "project"}
1782
1783         exited = arvados_cwl.main(
1784             ["--create-workflow", "--debug",
1785              "--api=containers",
1786              "--project-uuid", project_uuid,
1787              "tests/wf/revsort/revsort.cwl"],
1788             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
1789
1790         stubs.api.pipeline_templates().create.refute_called()
1791         stubs.api.container_requests().create.refute_called()
1792
1793         self.assertEqual(stubs.capture_stdout.getvalue(),
1794                          stubs.expect_workflow_uuid + '\n')
1795         self.assertEqual(exited, 0)